Monitoring với Bash: Báo cáo hệ thống và cảnh báo tự động trong DevOps

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# CPU usage từ idle
cpu_usage() {
    top -bn1 | grep "Cpu(s)" | awk '{print 100 - $8}' | tr -d ','
}

# Load average 1 phút
load_avg() {
    awk '{print $1}' /proc/loadavg
}

# Số CPU cores
cpu_cores() {
    nproc
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# RAM usage percentage
ram_usage() {
    free | awk '/Mem:/ {printf "%.1f", $3/$2*100}'
}

# RAM used (MB)
ram_used() {
    free -m | awk '/Mem:/ {print $3}'
}

# RAM total (MB)
ram_total() {
    free -m | awk '/Mem:/ {print $2}'
}

# Swap usage percentage
swap_usage() {
    free | awk '/Swap:/ {if ($2 > 0) printf "%.1f", $3/$2*100; else print "0"}'
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# Root partition usage percentage
disk_usage() {
    df / | tail -1 | awk '{print $5}' | tr -d '%'
}

# Disk used (GB)
disk_used() {
    df -BG / | tail -1 | awk '{print $3}' | tr -d 'G'
}

# Disk total (GB)
disk_total() {
    df -BG / | tail -1 | awk '{print $2}' | tr -d 'G'
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# Network bytes received (từ boot)
net_rx() {
    awk '/eth0|ens/ {print $2}' /proc/net/dev
}

# Network bytes transmitted (từ boot)
net_tx() {
    awk '/eth0|ens/ {print $10}' /proc/net/dev
}

# Số kết nối TCP đang established
tcp_connections() {
    ss -s | awk '/^TCP:/ {print $4}' | tr -d ','
}

1
2
3
4
5
6
7
8
9

# Số tiến trình đang chạy
running_procs() {
    ps aux | wc -l
}

# Uptime (giờ)
uptime_hours() {
    awk '{printf "%.1f", $1/3600}' /proc/uptime
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

send_slack() {
    local message="$1"
    local webhook="${SLACK_WEBHOOK:-}"

    if [[ -z "$webhook" ]]; then
        echo "[WARN] SLACK_WEBHOOK not set — skipping notification"
        return 1
    fi

    curl -s -X POST -H 'Content-type: application/json' \
        --data "{\"text\":\"${message}\"}" \
        "$webhook" >/dev/null 2>&1
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

send_telegram() {
    local message="$1"
    local bot_token="${TELEGRAM_BOT_TOKEN:-}"
    local chat_id="${TELEGRAM_CHAT_ID:-}"

    if [[ -z "$bot_token" || -z "$chat_id" ]]; then
        echo "[WARN] TELEGRAM_BOT_TOKEN or TELEGRAM_CHAT_ID not set"
        return 1
    fi

    curl -s -X POST "https://api.telegram.org/bot${bot_token}/sendMessage" \
        -d "chat_id=${chat_id}" \
        -d "text=${message}" \
        -d "parse_mode=Markdown" >/dev/null 2>&1
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

send_email() {
    local subject="$1"
    local body="$2"
    local recipient="${ALERT_EMAIL:-}"

    if [[ -z "$recipient" ]]; then
        echo "[WARN] ALERT_EMAIL not set"
        return 1
    fi

    echo "$body" | mail -s "$subject" "$recipient" 2>/dev/null
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

readonly CSV_FILE="${CSV_FILE:-/var/log/metrics.csv}"

# Tạo header nếu file chưa tồn tại
init_csv() {
    if [[ ! -f "$CSV_FILE" ]]; then
        echo "timestamp,hostname,cpu_pct,ram_pct,disk_pct,load_avg,procs,tcp_conn" > "$CSV_FILE"
    fi
}

# Ghi một dòng metric
record_metric() {
    local ts hostname cpu ram disk load procs conn
    ts=$(date '+%Y-%m-%d %H:%M:%S')
    hostname=$(hostname -s)
    cpu=$(cpu_usage)
    ram=$(ram_usage)
    disk=$(disk_usage)
    load=$(load_avg)
    procs=$(running_procs)
    conn=$(tcp_connections)

    echo "${ts},${hostname},${cpu},${ram},${disk},${load},${procs},${conn}" >> "$CSV_FILE"
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193

#!/usr/bin/env bash
# ==============================================================================
# Script : daily-report.sh
# Mục đích: Thu thập metric, tạo daily report, gửi alert khi vượt ngưỡng
# Chạy qua cron: 0 8 * * * /opt/scripts/daily-report.sh
# ==============================================================================

set -euo pipefail
IFS=$'\n\t'

readonly LOG_DIR="/var/log/daily-report"
readonly TIMESTAMP=$(date '+%Y%m%d_%H%M%S')
readonly REPORT_FILE="${LOG_DIR}/report_${TIMESTAMP}.txt"
readonly CSV_FILE="${LOG_DIR}/metrics.csv"
readonly HOSTNAME=$(hostname -s)
readonly DATE_STR=$(date '+%Y-%m-%d %H:%M:%S')

# Thresholds
readonly CPU_WARN="${CPU_WARN:-80}"
readonly CPU_CRIT="${CPU_CRIT:-95}"
readonly RAM_WARN="${RAM_WARN:-80}"
readonly RAM_CRIT="${RAM_CRIT:-95}"
readonly DISK_WARN="${DISK_WARN:-80}"
readonly DISK_CRIT="${DISK_CRIT:-90}"

mkdir -p "$LOG_DIR"

# --- Metric collection functions ---

cpu_usage() {
    top -bn1 | grep "Cpu(s)" | awk '{printf "%.1f", 100 - $8}' | tr -d ','
}

ram_usage() {
    free | awk '/Mem:/ {printf "%.1f", $3/$2*100}'
}

ram_used_mb() {
    free -m | awk '/Mem:/ {print $3}'
}

disk_usage() {
    df / | tail -1 | awk '{print $5}' | tr -d '%'
}

disk_used_gb() {
    df -BG / | tail -1 | awk '{print $3}' | tr -d 'G'
}

load_avg() {
    awk '{print $1}' /proc/loadavg
}

tcp_connections() {
    ss -s | awk '/^TCP:/ {print $4}' | tr -d ','
}

running_procs() {
    ps aux | wc -l
}

uptime_display() {
    uptime -p 2>/dev/null || awk '{d=int($1/86400); h=int(($1%86400)/3600); m=int(($1%3600)/60); printf "up %dd %dh %dm", d, h, m}' /proc/uptime
}

# --- CSV recording ---

init_csv() {
    if [[ ! -f "$CSV_FILE" ]]; then
        echo "timestamp,hostname,cpu_pct,ram_pct,disk_pct,load_avg,procs,tcp_conn" > "$CSV_FILE"
    fi
}

record_csv() {
    local cpu="$1" ram="$2" disk="$3" load="$4" procs="$5" conn="$6"
    echo "${DATE_STR},${HOSTNAME},${cpu},${ram},${disk},${load},${procs},${conn}" >> "$CSV_FILE"
}

# --- Alert functions ---

determine_level() {
    local value="$1" warn="$2" crit="$3"
    if [[ "$(echo "$value >= $crit" | bc 2>/dev/null || echo 0)" -eq 1 ]]; then
        echo "CRITICAL"
    elif [[ "$(echo "$value >= $warn" | bc 2>/dev/null || echo 0)" -eq 1 ]]; then
        echo "WARNING"
    else
        echo "OK"
    fi
}

send_slack() {
    local msg="$1"
    if [[ -n "${SLACK_WEBHOOK:-}" ]]; then
        curl -s -X POST -H 'Content-type: application/json' \
            --data "{\"text\":\"${msg}\"}" "$SLACK_WEBHOOK" >/dev/null 2>&1
    fi
}

send_telegram() {
    local msg="$1"
    if [[ -n "${TELEGRAM_BOT_TOKEN:-}" && -n "${TELEGRAM_CHAT_ID:-}" ]]; then
        curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
            -d "chat_id=${TELEGRAM_CHAT_ID}" -d "text=${msg}" -d "parse_mode=Markdown" >/dev/null 2>&1
    fi
}

notify() {
    local msg="$1"
    send_slack "$msg"
    send_telegram "$msg"
}

# --- Main report generation ---

generate_report() {
    local cpu ram disk load procs conn uptime ram_used disk_used

    cpu=$(cpu_usage)
    ram=$(ram_usage)
    disk=$(disk_usage)
    load=$(load_avg)
    procs=$(running_procs)
    conn=$(tcp_connections)
    uptime_display=$(uptime_display)
    ram_used=$(ram_used_mb)
    disk_used=$(disk_used_gb)

    # Write report file
    {
        echo "========================================"
        echo "  DAILY SYSTEM REPORT"
        echo "  Host: ${HOSTNAME}"
        echo "  Date: ${DATE_STR}"
        echo "========================================"
        echo ""
        echo "--- System Info ---"
        echo "  Uptime:          ${uptime_display}"
        echo "  Processes:       ${procs}"
        echo "  TCP Connections: ${conn}"
        echo ""
        echo "--- Resource Usage ---"
        echo "  CPU:    ${cpu}%"
        echo "  RAM:    ${ram}% (${ram_used} MB used)"
        echo "  Disk:   ${disk}% (${disk_used} GB used)"
        echo "  Load:   ${load}"
        echo ""
        echo "--- Thresholds ---"
        echo "  CPU:    $(determine_level "$cpu" "$CPU_WARN" "$CPU_CRIT")"
        echo "  RAM:    $(determine_level "$ram" "$RAM_WARN" "$RAM_CRIT")"
        echo "  Disk:   $(determine_level "$disk" "$DISK_WARN" "$DISK_CRIT")"
        echo ""
        echo "--- Top 5 CPU Processes ---"
        ps aux --sort=-%cpu | head -6 | awk '{printf "  %-8s %5s%% CPU  %5s%% RAM  %s\n", $1, $3, $4, $11}'
        echo ""
        echo "--- Top 5 RAM Processes ---"
        ps aux --sort=-%mem | head -6 | awk '{printf "  %-8s %5s%% CPU  %5s%% RAM  %s\n", $1, $3, $4, $11}'
        echo ""
        echo "========================================"
    } > "$REPORT_FILE"

    # Record to CSV
    init_csv
    record_csv "$cpu" "$ram" "$disk" "$load" "$procs" "$conn"

    # Print report to stdout
    cat "$REPORT_FILE"

    # Send alerts if needed
    local alerts=""
    local cpu_level ram_level disk_level
    cpu_level=$(determine_level "$cpu" "$CPU_WARN" "$CPU_CRIT")
    ram_level=$(determine_level "$ram" "$RAM_WARN" "$RAM_CRIT")
    disk_level=$(determine_level "$disk" "$DISK_WARN" "$DISK_CRIT")

    [[ "$cpu_level" != "OK" ]] && alerts+="CPU: ${cpu}% (${cpu_level})\n"
    [[ "$ram_level" != "OK" ]] && alerts+="RAM: ${ram}% (${ram_level})\n"
    [[ "$disk_level" != "OK" ]] && alerts+="Disk: ${disk}% (${disk_level})\n"

    if [[ -n "$alerts" ]]; then
        local alert_msg="*[${HOSTNAME}] System Alert*\n${alerts}Uptime: ${uptime_display}"
        notify -e "$alert_msg"
    fi
}

main() {
    generate_report
    echo ""
    echo "Report saved to: $REPORT_FILE"
    echo "CSV log: $CSV_FILE"
}

main "$@"

1
2

# Chạy hàng ngày lúc 8:00 sáng
0 8 * * * /opt/scripts/daily-report.sh >> /var/log/daily-report/cron.log 2>&1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

========================================
  DAILY SYSTEM REPORT
  Host: web-prod-01
  Date: 2026-09-01 08:00:00
========================================

--- System Info ---
  Uptime:          up 45 days 3 hours 22 minutes
  Processes:       234
  TCP Connections: 187

--- Resource Usage ---
  CPU:    23.4%
  RAM:    67.2% (5412 MB used)
  Disk:   72% (58 GB used)
  Load:   1.85

--- Thresholds ---
  CPU:    OK
  RAM:    OK
  Disk:    OK

--- Top 5 CPU Processes ---
  root       12.3% CPU   0.1% RAM  nginx: worker
  www-data    8.7% CPU   2.1% RAM  php-fpm: pool
  mysql       5.2% CPU  15.3% RAM  mysqld
  root        3.1% CPU   0.8% RAM  node /opt/api
  root        1.4% CPU   0.2% RAM  sshd

--- Top 5 RAM Processes ---
  mysql      15.3% RAM   5.2% CPU  mysqld
  www-data    2.1% RAM   8.7% CPU  php-fpm: pool
  root        1.8% RAM   0.9% CPU  node /opt/api
  root        0.8% RAM   3.1% CPU  node /opt/api
  redis       0.5% RAM   0.3% CPU  redis-server
========================================