# Prometheus 다운로드
wget https://github.com/prometheus/prometheus/releases/download/v2.45.0/prometheus-2.45.0.linux-amd64.tar.gz

# 압축 해제
tar xvfz prometheus-2.45.0.linux-amd64.tar.gz

# 디렉토리 이동
cd prometheus-2.45.0.linux-amd64

# Prometheus 실행
./prometheus --config.file=prometheus.yml

global:
  scrape_interval: 15s      # 기본 스크래핑 간격
  evaluation_interval: 15s  # 규칙 평가 간격
  
  external_labels:
    cluster: 'production'
    region: 'asia-northeast-2'

# Alertmanager 설정
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - 'localhost:9093'

# 알림 규칙 파일 로드
rule_files:
  - "alerts.yml"
  - "recording_rules.yml"

# 스크래핑 타겟 설정
scrape_configs:
  # Prometheus 자체 모니터링
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
  
  # Node Exporter 모니터링
  - job_name: 'node'
    static_configs:
      - targets: 
        - 'server1:9100'
        - 'server2:9100'
        - 'server3:9100'
        labels:
          env: 'production'
  
  # 애플리케이션 모니터링
  - job_name: 'my-app'
    scrape_interval: 5s  # 이 job만 5초 간격
    static_configs:
      - targets: ['app1:8080', 'app2:8080']
        labels:
          service: 'api'
          version: 'v2.0'

# Kubernetes Service Discovery 예제
scrape_configs:
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__

# AWS EC2 Service Discovery 예제
  - job_name: 'ec2-nodes'
    ec2_sd_configs:
      - region: ap-northeast-2
        access_key: YOUR_ACCESS_KEY
        secret_key: YOUR_SECRET_KEY
        port: 9100
    relabel_configs:
      - source_labels: [__meta_ec2_tag_Environment]
        target_label: environment

# Node Exporter 다운로드 및 설치
wget https://github.com/prometheus/node_exporter/releases/download/v1.6.0/node_exporter-1.6.0.linux-amd64.tar.gz
tar xvfz node_exporter-1.6.0.linux-amd64.tar.gz
cd node_exporter-1.6.0.linux-amd64

# Node Exporter 실행
./node_exporter

# systemd 서비스로 등록 (권장)
sudo tee /etc/systemd/system/node_exporter.service << EOF
[Unit]
Description=Node Exporter
After=network.target

[Service]
Type=simple
ExecStart=/usr/local/bin/node_exporter
Restart=always

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload
sudo systemctl start node_exporter
sudo systemctl enable node_exporter

# CPU 사용률 계산 (idle 모드를 제외한 비율)
100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# 메모리 사용률 계산
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100

# 디스크 사용률 계산
(node_filesystem_size_bytes - node_filesystem_avail_bytes) / node_filesystem_size_bytes * 100

# 네트워크 트래픽 속도 (초당 바이트)
rate(node_network_receive_bytes_total[5m])

# HTTP 요청 성공률
sum(rate(http_requests_total{status=~"2.."}[5m])) / sum(rate(http_requests_total[5m])) * 100

# 95 percentile 응답 시간
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))

# Alertmanager 다운로드
wget https://github.com/prometheus/alertmanager/releases/download/v0.26.0/alertmanager-0.26.0.linux-amd64.tar.gz
tar xvfz alertmanager-0.26.0.linux-amd64.tar.gz
cd alertmanager-0.26.0.linux-amd64

# Alertmanager 실행
./alertmanager --config.file=alertmanager.yml

global:
  resolve_timeout: 5m
  slack_api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'

# 알림 라우팅 설정
route:
  group_by: ['alertname', 'cluster', 'service']
  group_wait: 10s        # 첫 알림을 보내기 전 대기 시간
  group_interval: 10s    # 같은 그룹의 추가 알림 간격
  repeat_interval: 12h   # 알림 반복 간격
  receiver: 'default'
  
  routes:
    # 심각한 알림은 즉시 PagerDuty로
    - match:
        severity: critical
      receiver: 'pagerduty'
      continue: true
    
    # 데이터베이스 관련 알림은 DBA 팀으로
    - match_re:
        service: mysql|postgresql
      receiver: 'dba-team'
    
    # 개발 환경 알림은 Slack으로만
    - match:
        environment: development
      receiver: 'slack-dev'

# 수신자 설정
receivers:
  - name: 'default'
    slack_configs:
      - channel: '#alerts'
        title: '{{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'
  
  - name: 'pagerduty'
    pagerduty_configs:
      - service_key: 'YOUR_PAGERDUTY_KEY'
  
  - name: 'dba-team'
    email_configs:
      - to: 'dba-team@company.com'
        from: 'alertmanager@company.com'
        smarthost: 'smtp.gmail.com:587'
        auth_username: 'alertmanager@company.com'
        auth_password: 'your-password'
    slack_configs:
      - channel: '#dba-alerts'
  
  - name: 'slack-dev'
    slack_configs:
      - channel: '#dev-alerts'
        send_resolved: true

# 알림 억제 규칙
inhibit_rules:
  # 노드가 다운되면 해당 노드의 다른 알림은 억제
  - source_match:
      severity: 'critical'
      alertname: 'NodeDown'
    target_match:
      severity: 'warning'
    equal: ['instance']

groups:
  - name: node_alerts
    interval: 30s
    rules:
      # 인스턴스 다운 알림
      - alert: InstanceDown
        expr: up == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Instance {{ $labels.instance }} down"
          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
      
      # CPU 사용률 높음
      - alert: HighCPUUsage
        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "High CPU usage on {{ $labels.instance }}"
          description: "CPU usage is above 80% (current value: {{ $value }}%)"
      
      # 메모리 부족
      - alert: LowMemory
        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 < 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Low memory on {{ $labels.instance }}"
          description: "Available memory is below 10% (current: {{ $value }}%)"
      
      # 디스크 공간 부족
      - alert: DiskSpaceLow
        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 15
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Low disk space on {{ $labels.instance }}"
          description: "Disk space is below 15% (current: {{ $value }}%)"
      
      # 디스크 공간 심각
      - alert: DiskSpaceCritical
        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 5
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Critical disk space on {{ $labels.instance }}"
          description: "Disk space is critically low at {{ $value }}%"

  - name: application_alerts
    interval: 30s
    rules:
      # HTTP 에러율 높음
      - alert: HighErrorRate
        expr: (sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m]))) * 100 > 5
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High HTTP error rate"
          description: "Error rate is above 5% (current: {{ $value }}%)"
      
      # 응답 시간 느림
      - alert: SlowResponseTime
        expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 2
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Slow response time"
          description: "95th percentile response time is above 2s (current: {{ $value }}s)"
      
      # 데이터베이스 연결 실패
      - alert: DatabaseConnectionFailed
        expr: mysql_up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Database connection failed"
          description: "Cannot connect to database {{ $labels.instance }}"

# Ubuntu/Debian
sudo apt-get install -y software-properties-common
sudo add-apt-repository "deb https://packages.grafana.com/oss/deb stable main"
wget -q -O - https://packages.grafana.com/gpg.key | sudo apt-key add -
sudo apt-get update
sudo apt-get install grafana

# CentOS/RHEL
sudo tee /etc/yum.repos.d/grafana.repo << EOF
[grafana]
name=grafana
baseurl=https://packages.grafana.com/oss/rpm
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://packages.grafana.com/gpg.key
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
EOF

sudo yum install grafana

# 서비스 시작
sudo systemctl start grafana-server
sudo systemctl enable grafana-server

# 총 요청 수
sum(increase(http_requests_total[24h]))

# 평균 응답 시간
avg(rate(http_request_duration_seconds_sum[5m]) / rate(http_request_duration_seconds_count[5m]))

# 에러율
(sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m]))) * 100

# 초당 요청 수
sum(rate(http_requests_total[5m]))

# HTTP 상태 코드별 요청 수
sum by (status) (rate(http_requests_total[5m]))

# 엔드포인트별 요청 수
topk(10, sum by (endpoint) (rate(http_requests_total[5m])))

# 95th percentile 응답 시간
histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))

# 99th percentile 응답 시간
histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))

# CPU 사용률 (서버별)
100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# 메모리 사용률
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100

# 디스크 사용률
(node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_avail_bytes{mountpoint="/"}) / node_filesystem_size_bytes{mountpoint="/"} * 100

# 네트워크 수신 속도 (Mbps)
rate(node_network_receive_bytes_total[5m]) * 8 / 1000000

# 네트워크 송신 속도 (Mbps)
rate(node_network_transmit_bytes_total[5m]) * 8 / 1000000

# 디스크 I/O 사용률
rate(node_disk_io_time_seconds_total[5m]) * 100

# 시스템 로드 평균
node_load1
node_load5
node_load15

# 프로세스 수
node_procs_running

# 파일 디스크립터 사용률
(node_filefd_allocated / node_filefd_maximum) * 100

groups:
  - name: recording_rules
    interval: 30s
    rules:
      # CPU 사용률을 미리 계산
      - record: instance:node_cpu_utilization:rate5m
        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
      
      # 메모리 사용률을 미리 계산
      - record: instance:node_memory_utilization:ratio
        expr: 1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)
      
      # HTTP 요청 성공률을 미리 계산
      - record: job:http_requests:success_rate5m
        expr: sum by (job) (rate(http_requests_total{status=~"2.."}[5m])) / sum by (job) (rate(http_requests_total[5m]))
      
      # 서비스별 평균 응답 시간
      - record: service:http_request_duration:avg5m
        expr: sum by (service) (rate(http_request_duration_seconds_sum[5m])) / sum by (service) (rate(http_request_duration_seconds_count[5m]))

version: '3.8'

services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./alerts.yml:/etc/prometheus/alerts.yml
      - prometheus-data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
      - '--web.console.templates=/usr/share/prometheus/consoles'
    ports:
      - "9090:9090"
    networks:
      - monitoring
    restart: unless-stopped

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    volumes:
      - grafana-data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_USERS_ALLOW_SIGN_UP=false
    ports:
      - "3000:3000"
    networks:
      - monitoring
    restart: unless-stopped
    depends_on:
      - prometheus

  alertmanager:
    image: prom/alertmanager:latest
    container_name: alertmanager
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml
      - alertmanager-data:/alertmanager
    command:
      - '--config.file=/etc/alertmanager/alertmanager.yml'
      - '--storage.path=/alertmanager'
    ports:
      - "9093:9093"
    networks:
      - monitoring
    restart: unless-stopped

  node-exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    ports:
      - "9100:9100"
    networks:
      - monitoring
    restart: unless-stopped

  cadvisor:
    image: gcr.io/cadvisor/cadvisor:latest
    container_name: cadvisor
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
    ports:
      - "8080:8080"
    networks:
      - monitoring
    restart: unless-stopped

networks:
  monitoring:
    driver: bridge

volumes:
  prometheus-data:
  grafana-data:
  alertmanager-data:

# Helm 저장소 추가
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

# kube-prometheus-stack 설치
helm install prometheus prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --create-namespace \
  --set prometheus.prometheusSpec.retention=30d \
  --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage=50Gi \
  --set grafana.adminPassword=admin123

# 설치 확인
kubectl get pods -n monitoring