# Linux 환경에서 다운로드 및 설치
wget https://github.com/prometheus/prometheus/releases/download/v2.45.0/prometheus-2.45.0.linux-amd64.tar.gz
tar xvfz prometheus-2.45.0.linux-amd64.tar.gz
cd prometheus-2.45.0.linux-amd64

# Prometheus 실행
./prometheus --config.file=prometheus.yml

# Docker로 Prometheus 실행
docker run -d \
  --name prometheus \
  -p 9090:9090 \
  -v /path/to/prometheus.yml:/etc/prometheus/prometheus.yml \
  prom/prometheus

# 실행 확인
docker ps | grep prometheus

# prometheus.yml 기본 구조
global:
  scrape_interval: 15s      # 메트릭 수집 주기 (기본 15초)
  evaluation_interval: 15s   # 규칙 평가 주기
  
  external_labels:
    cluster: 'my-cluster'
    region: 'ap-northeast-2'

# 알림 규칙 파일 경로
rule_files:
  - "alerts.yml"

# 모니터링 대상 설정
scrape_configs:
  # Prometheus 자체 모니터링
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
  
  # Node Exporter (서버 메트릭)
  - job_name: 'node'
    static_configs:
      - targets: ['localhost:9100']
        labels:
          env: 'production'
          team: 'backend'

# Node Exporter 다운로드 및 설치
wget https://github.com/prometheus/node_exporter/releases/download/v1.6.0/node_exporter-1.6.0.linux-amd64.tar.gz
tar xvfz node_exporter-1.6.0.linux-amd64.tar.gz
cd node_exporter-1.6.0.linux-amd64

# 백그라운드로 실행
nohup ./node_exporter &

# 메트릭 확인 (브라우저나 curl로)
curl http://localhost:9100/metrics

# Grafana GPG 키 추가
wget -q -O - https://packages.grafana.com/gpg.key | sudo apt-key add -

# Grafana 저장소 추가
echo "deb https://packages.grafana.com/oss/deb stable main" | sudo tee /etc/apt/sources.list.d/grafana.list

# 설치
sudo apt-get update
sudo apt-get install grafana

# 서비스 시작 및 자동 시작 설정
sudo systemctl start grafana-server
sudo systemctl enable grafana-server

# 상태 확인
sudo systemctl status grafana-server

# Docker로 Grafana 실행
docker run -d \
  --name=grafana \
  -p 3000:3000 \
  -v grafana-storage:/var/lib/grafana \
  grafana/grafana-oss

# 실행 확인
docker logs grafana

Name: Prometheus
URL: http://localhost:9090
Access: Server (default)

# 고급 설정 (선택사항)
Scrape interval: 15s
Query timeout: 60s

# CPU 사용률 (%)
100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# 단순 메트릭 조회
node_cpu_seconds_total

# 레이블로 필터링
node_cpu_seconds_total{mode="idle"}

# 여러 조건 조합
node_cpu_seconds_total{mode="idle", cpu="0"}

# 최근 5분간의 데이터
node_cpu_seconds_total[5m]

# 최근 1시간
node_cpu_seconds_total[1h]

# 최근 1일
node_cpu_seconds_total[1d]

# 평균
avg(node_cpu_seconds_total)

# 합계
sum(node_cpu_seconds_total)

# 최대값
max(node_cpu_seconds_total)

# 최소값
min(node_cpu_seconds_total)

# 개수
count(node_cpu_seconds_total)

# 임계값 기반 색상 설정 예시
Thresholds:
  - 0: Green (#73BF69)      → 정상
  - 70: Yellow (#FADE2A)    → 주의
  - 85: Orange (#FF9830)    → 경고
  - 95: Red (#F2495C)       → 위험

# 그라디언트 색상 (시계열 그래프)
Color scheme: Green-Yellow-Red
Gradient mode: Scheme

# 서버 인스턴스 선택 변수
Name: instance
Type: Query
Data source: Prometheus
Query: label_values(node_cpu_seconds_total, instance)
Multi-value: true
Include All option: true

# 환경 선택 변수
Name: env
Type: Query
Query: label_values(node_cpu_seconds_total, env)
Multi-value: false

# 변수를 사용한 쿼리
node_cpu_seconds_total{instance="$instance", env="$env"}

# 정규표현식으로 여러 값 매칭
node_cpu_seconds_total{instance=~"$instance"}

# Alertmanager 다운로드 및 설치
wget https://github.com/prometheus/alertmanager/releases/download/v0.26.0/alertmanager-0.26.0.linux-amd64.tar.gz
tar xvfz alertmanager-0.26.0.linux-amd64.tar.gz
cd alertmanager-0.26.0.linux-amd64

# 설정 파일 생성 (alertmanager.yml)
cat > alertmanager.yml << EOF
global:
  resolve_timeout: 5m
  
route:
  group_by: ['alertname', 'cluster']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 12h
  receiver: 'default'
  
receivers:
  - name: 'default'
    slack_configs:
      - api_url: 'YOUR_SLACK_WEBHOOK_URL'
        channel: '#alerts'
        title: '🚨 Alert: {{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'
EOF

# Alertmanager 실행
./alertmanager --config.file=alertmanager.yml

groups:
  - name: system_alerts
    interval: 30s
    rules:
      # CPU 사용률 높음
      - alert: HighCPUUsage
        expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High CPU usage on {{ $labels.instance }}"
          description: "CPU usage is {{ $value }}% for 5 minutes"
      
      # 메모리 부족
      - alert: HighMemoryUsage
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High memory usage on {{ $labels.instance }}"
          description: "Memory usage is {{ $value }}%"
      
      # 디스크 공간 부족
      - alert: DiskSpaceLow
        expr: (1 - (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"})) * 100 > 85
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Low disk space on {{ $labels.instance }}"
          description: "Disk usage is {{ $value }}%"
      
      # 서버 다운
      - alert: InstanceDown
        expr: up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Instance {{ $labels.instance }} is down"
          description: "{{ $labels.instance }} has been down for more than 1 minute"

# prometheus.yml에 추가
rule_files:
  - "alerts.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - localhost:9093

# Slack 연동 예시
Name: Slack Alerts
Integration: Slack
Webhook URL: https://hooks.slack.com/services/YOUR/WEBHOOK/URL
Username: Grafana Bot
Channel: #monitoring

# 이메일 연동 예시
Name: Email Alerts
Integration: Email
Addresses: admin@example.com, team@example.com

# Pod CPU 사용률
sum(rate(container_cpu_usage_seconds_total{pod!=""}[5m])) by (pod)

# Pod 메모리 사용량
sum(container_memory_usage_bytes{pod!=""}) by (pod) / 1024 / 1024 / 1024

# Pod 재시작 횟수
kube_pod_container_status_restarts_total

# 노드 상태
kube_node_status_condition{condition="Ready", status="true"}

# Pending 상태 Pod
count(kube_pod_status_phase{phase="Pending"})

# recording_rules.yml
groups:
  - name: cpu_rules
    interval: 30s
    rules:
      # CPU 사용률을 미리 계산
      - record: instance:cpu_usage:rate5m
        expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
      
      # 서버별 평균 응답 시간
      - record: instance:http_response_time:avg
        expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))
      
      # 일별 요청 수 집계
      - record: job:http_requests:rate1h
        expr: sum by(job) (rate(http_requests_total[1h]))

# custom_exporter.py
from prometheus_client import start_http_server, Gauge
import time
import psutil

# 메트릭 정의
cpu_usage = Gauge('custom_cpu_usage', 'CPU usage percentage')
memory_usage = Gauge('custom_memory_usage', 'Memory usage percentage')
disk_usage = Gauge('custom_disk_usage', 'Disk usage percentage', ['partition'])

def collect_metrics():
    """시스템 메트릭 수집"""
    while True:
        # CPU 사용률
        cpu_usage.set(psutil.cpu_percent(interval=1))
        
        # 메모리 사용률
        memory = psutil.virtual_memory()
        memory_usage.set(memory.percent)
        
        # 디스크 사용률 (파티션별)
        for partition in psutil.disk_partitions():
            try:
                usage = psutil.disk_usage(partition.mountpoint)
                disk_usage.labels(partition=partition.device).set(usage.percent)
            except:
                pass
        
        time.sleep(15)  # 15초마다 수집

if __name__ == '__main__':
    # 8000 포트에서 메트릭 제공
    start_http_server(8000)
    print("Exporter running on port 8000")
    collect_metrics()

# Kubernetes Service Discovery 설정
scrape_configs:
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
      - role: pod
    
    relabel_configs:
      # prometheus.io/scrape 어노테이션이 있는 Pod만 수집
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      
      # 포트 설정
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: (.+)
        replacement: $1
      
      # 메트릭 경로 설정
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      
      # 레이블 추가
      - source_labels: [__meta_kubernetes_namespace]
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_pod_name]
        target_label: kubernetes_pod_name

# 플러그인 설치 명령어
grafana-cli plugins install grafana-worldmap-panel
grafana-cli plugins install natel-plotly-panel
grafana-cli plugins install camptocamp-prometheus-alertmanager-datasource

# 설치 후 Grafana 재시작
sudo systemctl restart grafana-server

# 중앙 Prometheus 설정
scrape_configs:
  - job_name: 'federate'
    scrape_interval: 15s
    honor_labels: true
    metrics_path: '/federate'
    
    params:
      'match[]':
        - '{job="prometheus"}'
        - '{__name__=~"job:.*"}'  # Recording rules
    
    static_configs:
      - targets:
        - 'prometheus-dc1:9090'
        - 'prometheus-dc2:9090'
        - 'prometheus-dc3:9090'