日志管理与分析

┌─────────────────────────────────────────────────────────────┐
│                    Linux 日志系统                           │
├─────────────────────────────────────────────────────────────┤
│                                                             │
│   应用 ──→ syslog/journlad ──→ /var/log/*                 │
│                    │                                        │
│         ┌─────────┴─────────┐                              │
│         ↓                   ↓                               │
│     journalctl          rsyslog                            │
│         │                   │                               │
│      二进制日志           文本日志                          │
│                                                             │
└─────────────────────────────────────────────────────────────┘

# /etc/rsyslog.conf

# 模块
module(load="imuxsock")        # 本地日志
module(load="imklog")         # 内核日志
module(load="imudp")          # UDP接收
module(load="imtcp")          # TCP接收

# 规则格式
# facility.priority   action

# facility: auth, authpriv, cron, daemon, kern, lpr, mail, news, syslog, user, uucp, local0-7
# priority: debug, info, notice, warning, warn, err, error, crit, alert, emerg

# 示例规则
*.info;mail.none;authpriv.none;cron.none    /var/log/messages
authpriv.*                                      /var/log/secure
mail.*                                          -/var/log/maillog
cron.*                                          /var/log/cron
*.emerg                                         :omusrmsg:*

# 发送端 - /etc/rsyslog.conf
*.* @@logserver.example.com:514

# 接收端配置
# /etc/rsyslog.conf
module(load="imudp")
input(type="imudp" port="514")

module(load="imtcp")
input(type="imtcp" port="514")

# 模板（格式化日志）
template(name="RemoteLogs" type="string" string="/var/log/remote/%HOSTNAME%/%PROGRAMNAME%.log")

# 应用模板
*.* ?RemoteLogs

# 重启服务
sudo systemctl restart rsyslog

# 查看日志
journalctl                          # 所有日志
journalctl -xe                     # 带上下文
journalctl -f                      # 实时跟踪
journalctl -b                      # 本次启动
journalctl -b -1                   # 上次启动
journalctl --since "1 hour ago"
journalctl --since "2026-01-01" --until "2026-01-02"
journalctl -u nginx.service        # 指定服务
journalctl -u nginx.service -u mysql.service
journalctl -p err                  # 指定级别

# 分页显示
journalctl --no-pager

# 显示最新
journalctl -n 50                   # 最近50条
journalctl -n 50 --no-pager

# 显示完整消息
journalctl -o verbose

# 磁盘使用
journalctl --disk-usage

# 清理旧日志
journalctl --vacuum-time=7d       # 保留7天
journalctl --vacuum-size=500M      # 保留500M

# /etc/systemd/journald.conf

[Journal]
Storage=persistent                 # 持久化存储
# Storage=volatile                  # 内存存储
# Storage=none                       # 不存储

Compress=yes
SystemMaxUse=500M                  # 最大磁盘占用
SystemMaxFileSize=50M              # 单文件最大
MaxRetentionSec=30day              # 最大保留时间

RateLimitInterval=1s
RateLimitBurst=1000

ForwardToSyslog=yes                # 转发到rsyslog
ForwardToKMsg=no
ForwardToConsole=no

MaxLevelKMsg=warning
MaxLevelSyslog=debug
MaxLevelConsole=info
MaxLevelStore=warning

# 常见日志文件
/var/log/messages           # 通用系统消息
/var/log/syslog            # 系统日志
/var/log/auth.log          # 认证日志
/var/log/secure            # 安全日志
/var/log/maillog           # 邮件日志
/var/log/cron             # 计划任务日志
/var/log/dmesg             # 启动时内核消息
/var/log/kern.log          # 内核日志
/var/log/dpkg.log          # 包安装日志
/var/log/yum.log           # yum日志

# 应用日志
/var/log/nginx/access.log
/var/log/nginx/error.log
/var/log/apache2/access.log
/var/log/apache2/error.log
/var/log/mysql/error.log
/var/log/mysql/slow.log

# 安装
sudo apt install logwatch

# 配置
sudo vim /etc/logwatch/conf/logwatch.conf

# 每天报告
sudo cp /usr/share/logwatch/default.conf/logwatch.conf /etc/logwatch/conf/

# 运行
logwatch --output mail --mailto admin@example.com
logwatch --detail high --mailto admin@example.com
logwatch --service nginx --detail high

# /etc/logrotate.conf
# 全局配置
weekly
rotate 4
create
dateext
compress
include /etc/logrotate.d/

# 特定配置 - /etc/logrotate.d/nginx
/var/log/nginx/*.log {
    daily
    missingok
    rotate 14
    compress
    delaycompress
    notifempty
    create 0640 www-data adm
    sharedscripts
    postrotate
        [ -f /var/run/nginx.pid ] && kill -USR1 `cat /var/run/nginx.pid`
    endscript
}

# Apache访问统计
awk '{print $7}' /var/log/apache2/access.log | \
    sort | uniq -c | sort -rn | head -20

# Nginx错误分析
awk '$4 ~ /error/ {print $0}' /var/log/nginx/error.log

# SSH登录统计
grep "Accepted" /var/log/auth.log | \
    awk '{print $11}' | sort | uniq -c | sort -rn

# 失败登录尝试
grep "Failed password" /var/log/auth.log | \
    awk '{print $11, $13}' | sort | uniq -c | sort -rn | head -20

# tail -f
tail -f /var/log/syslog
tail -f /var/log/nginx/access.log

# 多文件监控
tail -f /var/log/syslog /var/log/nginx/error.log

# 过滤+监控
tail -f /var/log/syslog | grep -i error

# multitail
sudo apt install multitail
multitail /var/log/syslog /var/log/nginx/error.log

┌─────────────────────────────────────────────────────────────┐
│                    ELK架构                                  │
├─────────────────────────────────────────────────────────────┤
│                                                             │
│   应用 ──→ Filebeat ──→ Logstash ──→ Elasticsearch          │
│                                     ↓                       │
│                                     ↓                       │
│                                     ↓                       │
│                                  Kibana ←─ 查询展示         │
│                                                             │
└─────────────────────────────────────────────────────────────┘

# /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/*.log
    - /var/log/nginx/*.log
  fields:
    type: system
  fields_under_root: true

output.elasticsearch:
  hosts: ["elasticsearch:9200"]
  index: "logs-%{+yyyy.MM.dd}"

processors:
  - add_host_metadata:
      when.not.contains.tags: forwarded
  - add_cloud_metadata: ~

# promtail-local-config.yaml
server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
- job_name: system
  static_configs:
  - targets:
      - localhost
    labels:
      job: syslog
      __path__: /var/log/*log