1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

# 检查Nacos服务注册状态
curl -s http://nacos-server:8848/nacos/v1/ns/service/list?pageNo=1\&pageSize=100 | jq '.count'

# 检查各微服务进程是否存活
for svc in gateway auth data-service report-service ai-service; do
  pid=$(pgrep -f "$svc" | head -1)
  if [ -z "$pid" ]; then
    echo "[ALERT] $svc is DOWN"
  else
    echo "[OK] $svc is running (PID: $pid)"
  fi
done

# 检查HDFS存储空间使用率
hdfs dfs -du -h / | head -5
hdfs dfsadmin -report | grep "DFS Used%"

# 检查关键目录日志是否有ERROR
grep -c "ERROR" /data/logs/data-service/$(date +%Y-%m-%d).log
grep -c "ERROR" /data/logs/gateway/$(date +%Y-%m-%d).log

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# Prometheus Alertmanager 告警抑制配置示例
inhibit_rules:
  # 当 NameNode 宕机时，抑制所有 HDFS 相关告警
  - source_matchers:
      - alertname = "NameNodeDown"
    target_matchers:
      - severity =~ "P2|P3"
      - component = "hdfs"
    equal: ['cluster']

  # 当数据库不可达时，抑制该库所有连接相关告警
  - source_matchers:
      - alertname = "DatabaseUnreachable"
    target_matchers:
      - alertname =~ "SlowQuery|ConnectionPoolExhausted"
    equal: ['instance']

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# 查看微服务最近500行日志并过滤ERROR
tail -n 500 /data/logs/data-service/application.log | grep -A 20 "ERROR"

# 查看Nacos注册中心日志
tail -f /data/nacos/logs/nacos.log

# 检查数据库连接状态（MySQL）
mysql -e "SHOW PROCESSLIST;" | awk '{print $1, $2, $3, $4, $5, $6, $7}' | sort -k6 -rn | head -20

# 检查Oracle活跃会话
sqlplus / as sysdba <<EOF
SELECT s.sid, s.serial#, s.username, s.status, s.machine, s.program
FROM v\$session s WHERE s.status = 'ACTIVE' AND s.type != 'BACKGROUND';
EOF

# 查看Kafka消费组Lag
kafka-consumer-groups.sh --bootstrap-server kafka-broker:9092 \
  --describe --group data-consumer-group

# 查看Hive正在执行的任务
hive -e "SHOW INSTANCES;"

# JVM内存实时查看（使用Arthas）
java -jar arthas-boot.jar <PID>
# 进入Arthas后执行：
# memory    -- 查看内存分配
# dashboard -- 实时面板
# thread -n 5  -- CPU最高的5个线程

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

#!/bin/bash
# nacos_health_check.sh - Nacos服务健康检查脚本
# 用法: ./nacos_health_check.sh [nacos_addr] [namespace]

NACOS_ADDR=${1:-"http://nacos-server:8848"}
NAMESPACE=${2:-"public"}
EXPECTED_SERVICES=${3:-42}  # 预期注册服务数量基线
ALERT_WEBHOOK="https://hooks.dingtalk.com/xxx"
TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")

echo "=========================================="
echo "Nacos Health Check - $TIMESTAMP"
echo "=========================================="

# 1. 检查Nacos自身可用性
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$NACOS_ADDR/nacos/v1/ns/operator/servers")
if [ "$HTTP_CODE" != "200" ]; then
  echo "[CRITICAL] Nacos server is unreachable (HTTP $HTTP_CODE)"
  exit 1
fi
echo "[OK] Nacos server is reachable"

# 2. 获取已注册服务列表
SERVICE_LIST=$(curl -s "$NACOS_ADDR/nacos/v1/ns/service/list?pageNo=1&pageSize=200&namespaceId=$NAMESPACE")
SERVICE_COUNT=$(echo "$SERVICE_LIST" | jq -r '.count')
echo "[INFO] Registered services: $SERVICE_COUNT (baseline: $EXPECTED_SERVICES)"

if [ "$SERVICE_COUNT" -lt "$EXPECTED_SERVICES" ]; then
  echo "[WARNING] Service count below baseline! Missing: $((EXPECTED_SERVICES - SERVICE_COUNT))"
fi

# 3. 逐服务检查健康实例
UNHEALTHY_SERVICES=()
for svc in $(echo "$SERVICE_LIST" | jq -r '.doms[]'); do
  INSTANCES=$(curl -s "$NACOS_ADDR/nacos/v1/ns/instance/list?serviceName=$svc&namespaceId=$NAMESPACE")
  TOTAL=$(echo "$INSTANCES" | jq '.hosts | length')
  HEALTHY=$(echo "$INSTANCES" | jq '[.hosts[] | select(.healthy == true)] | length')

  if [ "$HEALTHY" -lt "$TOTAL" ]; then
    UNHEALTHY_SERVICES+=("$svc ($HEALTHY/$TOTAL healthy)")
    echo "[WARNING] $svc: $HEALTHY/$TOTAL instances healthy"
  fi
done

# 4. 输出汇总
if [ ${#UNHEALTHY_SERVICES[@]} -eq 0 ]; then
  echo "[OK] All services have healthy instances"
else
  echo ""
  echo "=== UNHEALTHY SERVICES ==="
  for svc in "${UNHEALTHY_SERVICES[@]}"; do
    echo "  - $svc"
  done

  # 发送告警
  curl -s -X POST "$ALERT_WEBHOOK" \
    -H "Content-Type: application/json" \
    -d "{\"msgtype\":\"text\",\"text\":{\"content\":\"[Nacos告警] 以下服务存在不健康实例:\n$(printf '%s\n' "${UNHEALTHY_SERVICES[@]}")\"}}"
fi

echo "=========================================="
echo "Check completed at $(date '+%Y-%m-%d %H:%M:%S')"

1
2

# crontab 配置
0 9 * * 1-5 /opt/scripts/nacos_health_check.sh >> /data/logs/inspection/nacos_$(date +\%Y\%m\%d).log 2>&1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# systemd 服务自愈配置示例
# /etc/systemd/system/data-service.service
[Unit]
Description=Data Service
After=network.target nacos.service

[Service]
Type=simple
ExecStart=/opt/data-service/bin/start.sh
Restart=always
RestartSec=10
StartLimitBurst=3
StartLimitInterval=300
# 5分钟内最多重启3次，防止无限重启

[Install]
WantedBy=multi-user.target

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

#!/bin/bash
# disk_auto_heal.sh - 磁盘空间自愈脚本
THRESHOLD=85
LOG_RETAIN_DAYS=7

USAGE=$(df -h /data | awk 'NR==2{print $5}' | tr -d '%')

if [ "$USAGE" -gt "$THRESHOLD" ]; then
  echo "[AUTO-HEAL] Disk usage ${USAGE}% > ${THRESHOLD}%, triggering cleanup..."

  # 清理过期日志
  find /data/logs -name "*.log" -mtime +$LOG_RETAIN_DAYS -delete
  find /data/logs -name "*.log.gz" -mtime +30 -delete

  # 清理Hive临时文件
  hdfs dfs -rm -r /tmp/hive/* 2>/dev/null

  # 清理YARN应用日志
  yarn logs -applicationId ALL 2>/dev/null | head -1
  hdfs dfs -rm -r /app-logs/*/logs/* 2>/dev/null

  NEW_USAGE=$(df -h /data | awk 'NR==2{print $5}' | tr -d '%')
  echo "[AUTO-HEAL] Disk usage after cleanup: ${NEW_USAGE}%"
fi

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# 1. 网络连通性检查
telnet db-host 3306    # MySQL
telnet db-host 5236    # 达梦
telnet db-host 1521    # Oracle
telnet db-host 54321   # 人大金仓

# 2. 数据库用户权限验证
# MySQL
mysql -h db-host -u data_platform -p -e "SHOW GRANTS;"

# 达梦
disql SYSDBA/password@db-host:5236 -e "SELECT * FROM DBA_ROLE_PRIVS WHERE GRANTEE='DATA_PLATFORM';"

# 3. 驱动版本兼容性检查
# 确认JDBC驱动版本与数据库版本匹配
# MySQL Connector/J 8.0.x → MySQL 8.0.x
# DM JDBC Driver → 对应DM7/DM8版本
# ojdbc8.jar → Oracle 12c+
# kingbase8-8.6.0.jar → KingbaseES V8

1
2
3
4
5

# 达梦许可证到期检查
disql SYSDBA/password -e "SELECT EXPIRED_DATE FROM V\$LICENSE;"

# 人大金仓许可证检查
ksql -U system -d dbname -c "SELECT * FROM sys_license_info();"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# 微服务JVM参数推荐配置
# 轻量服务（网关、认证服务）
JAVA_OPTS="-Xms512m -Xmx1024m -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m"

# 中等服务（数据服务、报表服务）
JAVA_OPTS="-Xms1024m -Xmx2048m -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m -XX:+UseG1GC"

# 重型服务（AI推理服务、BI分析服务）
JAVA_OPTS="-Xms2048m -Xmx4096m -XX:MetaspaceSize=512m -XX:MaxMetaspaceSize=1024m -XX:+UseG1GC -XX:MaxGCPauseMillis=200"

# 大数据组件（Spark Driver）
SPARK_DRIVER_MEMORY="4g"
SPARK_EXECUTOR_MEMORY="8g"
SPARK_EXECUTOR_CORES=4

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

# ~/.ssh/config - SSH连接配置示例
Host nacos-server
    HostName 192.168.1.101
    User ops
    Port 22
    IdentityFile ~/.ssh/id_rsa_ops
    ConnectTimeout 10
    ServerAliveInterval 60
    ServerAliveCountMax 3

Host hadoop-nn
    HostName 192.168.1.110
    User hdfs
    Port 22
    IdentityFile ~/.ssh/id_rsa_ops

Host hadoop-dn-*
    HostName 192.168.1.%h
    User hdfs
    Port 22
    IdentityFile ~/.ssh/id_rsa_ops

# 批量检查集群节点状态
for i in $(seq 110 120); do
  echo -n "192.168.1.$i: "
  ssh -o ConnectTimeout=5 hdfs@192.168.1.$i "uptime" 2>/dev/null || echo "UNREACHABLE"
done

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# HDFS健康状态检查
hdfs fsck / -files -blocks -locations 2>/dev/null | tail -20

# 查看HDFS各目录空间占用
hdfs dfs -du -h -s /user /tmp /data /app-logs 2>/dev/null

# YARN资源使用概览
yarn top -delay 5 2>/dev/null | head -30

# Spark历史任务状态
curl -s http://spark-history-server:18080/api/v1/applications?status=completed\&limit=10 | jq '.[].name'

# Hive Metastore数据库检查
mysql -e "SELECT COUNT(*) as table_count FROM TBLS;" metastore_db
mysql -e "SELECT COUNT(*) as partition_count FROM PARTITIONS;" metastore_db

# 检查NameNode HA状态
hdfs haadmin -getAllServiceState
# Expected: One active, one standby