import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class NewsCrawler {
    public static void main(String[] args) {
        try {
            String url = "https://news.example.com";
            Document doc = Jsoup.connect(url)
                    .userAgent("Mozilla/5.0") // User-Agent 설정
                    .timeout(5000) // 타임아웃 5초
                    .get();
            
            // CSS 셀렉터로 기사 제목 선택
            Elements headlines = doc.select("h2.article-title");
            
            System.out.println("총 " + headlines.size() + "개의 기사를 찾았습니다.\n");
            
            // 각 제목 출력
            for (Element headline : headlines) {
                String title = headline.text();
                String link = headline.select("a").attr("href");
                
                System.out.println("제목: " + title);
                System.out.println("링크: " + link);
                System.out.println("---");
            }
            
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

// 태그 선택
Elements divs = doc.select("div");

// 클래스 선택
Elements contents = doc.select(".content");

// ID 선택
Elements header = doc.select("#header");

// 속성 선택
Elements links = doc.select("a[href]");

// 속성 값으로 선택
Elements pdfLinks = doc.select("a[href$=.pdf]");

// 여러 조건 조합
Elements newsLinks = doc.select("div.news a[href]");

// 자식 선택 (직계 자식만)
Elements directChildren = doc.select("div > p");

// 후손 선택 (모든 하위 요소)
Elements allDescendants = doc.select("div p");

// 형제 선택 (바로 다음)
Elements nextSibling = doc.select("h2 + p");

// 형제 선택 (이후 모든)
Elements allSiblings = doc.select("h2 ~ p");

// 첫 번째 요소
Element first = doc.selectFirst("div.article");

// n번째 요소
Elements third = doc.select("li:nth-child(3)");

// 텍스트 포함 여부로 선택
Elements matching = doc.select("p:contains(검색어)");

// 정규식으로 선택
Elements regex = doc.select("a[href~=(?i)\\.(png|jpe?g)]");

// 부정 선택
Elements notAds = doc.select("div:not(.advertisement)");

import java.util.concurrent.TimeUnit;

public class PoliteCrawler {
    public static void main(String[] args) {
        String[] urls = {
            "https://example.com/page1",
            "https://example.com/page2",
            "https://example.com/page3"
        };
        
        for (String url : urls) {
            try {
                System.out.println("크롤링 중: " + url);
                Document doc = Jsoup.connect(url)
                        .userAgent("Mozilla/5.0")
                        .timeout(5000)
                        .get();
                
                // 데이터 처리
                processData(doc);
                
                // 2초 대기 (예의 바른 크롤링!)
                TimeUnit.SECONDS.sleep(2);
                
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                break;
            } catch (Exception e) {
                System.err.println("에러 발생: " + e.getMessage());
            }
        }
    }
    
    private static void processData(Document doc) {
        // 데이터 처리 로직
        System.out.println("제목: " + doc.title());
    }
}

public class RetryableCrawler {
    private static final int MAX_RETRIES = 3;
    private static final int RETRY_DELAY = 2000; // 2초
    
    public static Document fetchWithRetry(String url) {
        int attempts = 0;
        
        while (attempts < MAX_RETRIES) {
            try {
                return Jsoup.connect(url)
                        .userAgent("Mozilla/5.0")
                        .timeout(5000)
                        .get();
                        
            } catch (IOException e) {
                attempts++;
                System.err.println("시도 " + attempts + " 실패: " + e.getMessage());
                
                if (attempts >= MAX_RETRIES) {
                    System.err.println("최대 재시도 횟수 초과");
                    throw new RuntimeException("크롤링 실패: " + url, e);
                }
                
                try {
                    Thread.sleep(RETRY_DELAY);
                } catch (InterruptedException ie) {
                    Thread.currentThread().interrupt();
                    throw new RuntimeException("재시도 중 인터럽트 발생", ie);
                }
            }
        }
        
        return null; // 도달하지 않음
    }
}

import org.jsoup.Connection;
import java.util.Map;

public class LoginCrawler {
    public static void main(String[] args) {
        try {
            // 1단계: 로그인 페이지 접속 및 쿠키 획득
            Connection.Response loginForm = Jsoup.connect("https://example.com/login")
                    .method(Connection.Method.GET)
                    .execute();
            
            Map<string, string=""> cookies = loginForm.cookies();
            
            // 2단계: 로그인 요청 (POST)
            Connection.Response loginResponse = Jsoup.connect("https://example.com/login")
                    .data("username", "myuser")
                    .data("password", "mypass")
                    .cookies(cookies)
                    .method(Connection.Method.POST)
                    .execute();
            
            // 로그인 후 쿠키 업데이트
            cookies.putAll(loginResponse.cookies());
            
            // 3단계: 로그인이 필요한 페이지 접속
            Document doc = Jsoup.connect("https://example.com/protected-page")
                    .cookies(cookies)
                    .get();
            
            System.out.println("로그인 성공! 페이지 제목: " + doc.title());
            
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}</string,>

public class PaginationCrawler {
    public static void main(String[] args) {
        String baseUrl = "https://example.com/articles?page=";
        int maxPages = 10;
        
        for (int page = 1; page <= maxPages; page++) {
            try {
                String url = baseUrl + page;
                System.out.println("\n=== 페이지 " + page + " 크롤링 ===");
                
                Document doc = Jsoup.connect(url)
                        .userAgent("Mozilla/5.0")
                        .timeout(5000)
                        .get();
                
                // 기사 목록 추출
                Elements articles = doc.select("div.article");
                
                if (articles.isEmpty()) {
                    System.out.println("더 이상 기사가 없습니다.");
                    break; // 빈 페이지면 종료
                }
                
                for (Element article : articles) {
                    String title = article.select("h2").text();
                    String author = article.select(".author").text();
                    String date = article.select(".date").text();
                    
                    System.out.println("제목: " + title);
                    System.out.println("작성자: " + author);
                    System.out.println("날짜: " + date);
                    System.out.println("---");
                }
                
                // 예의 바른 크롤링을 위한 딜레이
                Thread.sleep(2000);
                
            } catch (Exception e) {
                System.err.println("페이지 " + page + " 크롤링 실패: " + e.getMessage());
            }
        }
    }
}

import java.io.FileWriter;
import java.io.IOException;

public class CsvSaver {
    public static void saveToCSV(List</code><article><code> articles, String filename) {
        try (FileWriter writer = new FileWriter(filename, StandardCharsets.UTF_8)) {
            // CSV 헤더 작성
            writer.append("제목,작성자,날짜,URL\n");
            
            // 데이터 작성
            for (Article article : articles) {
                writer.append(escapeCsv(article.getTitle())).append(",");
                writer.append(escapeCsv(article.getAuthor())).append(",");
                writer.append(escapeCsv(article.getDate())).append(",");
                writer.append(escapeCsv(article.getUrl())).append("\n");
            }
            
            System.out.println("CSV 파일 저장 완료: " + filename);
            
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    
    // CSV 특수문자 이스케이프 처리
    private static String escapeCsv(String value) {
        if (value == null) return "";
        
        if (value.contains(",") || value.contains("\"") || value.contains("\n")) {
            value = "\"" + value.replace("\"", "\"\"") + "\"";
        }
        
        return value;
    }
}

// Article 클래스
class Article {
    private String title;
    private String author;
    private String date;
    private String url;
    
    // 생성자, getter, setter 생략
}</code></article></pre>
</div>

<h3>🗄️ 데이터베이스에 저장하기</h3>

<p>
대량의 데이터를 체계적으로 관리하려면 데이터베이스가 최고야. JDBC를 사용한 간단한 예제를 볼까?
</p>

<div class="tx-code-block">
<pre><code>import java.sql.*;

public class DatabaseSaver {
    private static final String DB_URL = "jdbc:mysql://localhost:3306/crawling_db";
    private static final String USER = "username";
    private static final String PASS = "password";
    
    public static void saveToDatabase(List</code><article><code> articles) {
        String sql = "INSERT INTO articles (title, author, date, url, created_at) VALUES (?, ?, ?, ?, NOW())";
        
        try (Connection conn = DriverManager.getConnection(DB_URL, USER, PASS);
             PreparedStatement pstmt = conn.prepareStatement(sql)) {
            
            conn.setAutoCommit(false); // 트랜잭션 시작
            
            for (Article article : articles) {
                pstmt.setString(1, article.getTitle());
                pstmt.setString(2, article.getAuthor());
                pstmt.setString(3, article.getDate());
                pstmt.setString(4, article.getUrl());
                pstmt.addBatch();
            }
            
            int[] results = pstmt.executeBatch();
            conn.commit(); // 트랜잭션 커밋
            
            System.out.println(results.length + "개의 기사가 저장되었습니다.");
            
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }
}</code></article></pre>
</div>

<h3>📋 JSON 파일로 저장하기</h3>

<p>
API 형태로 데이터를 활용하려면 JSON이 좋아. Gson 라이브러리를 사용한 예제야.
</p>

<div class="tx-code-block">
<pre><code>import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import java.io.FileWriter;

public class JsonSaver {
    public static void saveToJson(List</code><article><code> articles, String filename) {
        Gson gson = new GsonBuilder()
                .setPrettyPrinting() // 보기 좋게 포맷팅
                .create();
        
        try (FileWriter writer = new FileWriter(filename, StandardCharsets.UTF_8)) {
            gson.toJson(articles, writer);
            System.out.println("JSON 파일 저장 완료: " + filename);
            
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}</code></article></pre>
</div>

<p>
Gson을 사용하려면 Maven 의존성을 추가해야 해:
</p>

<div class="tx-code-block">
<pre><code><dependency>
    <groupId>com.google.code.gson</groupId>
    <artifactId>gson</artifactId>
    <version>2.10.1</version>
</dependency>

// Selenium 예제 (참고용)
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;

WebDriver driver = new ChromeDriver();
driver.get("https://dynamic-site.com");

// JavaScript 실행 대기
Thread.sleep(3000);

// 렌더링된 HTML 가져오기
String html = driver.getPageSource();

// Jsoup으로 파싱
Document doc = Jsoup.parse(html);

driver.quit();

// API 직접 호출 예제
import java.net.http.*;

HttpClient client = HttpClient.newHttpClient();
HttpRequest request = HttpRequest.newBuilder()
        .uri(URI.create("https://api.example.com/articles"))
        .header("User-Agent", "Mozilla/5.0")
        .build();

HttpResponse<string> response = client.send(request, 
        HttpResponse.BodyHandlers.ofString());

// JSON 파싱
String jsonData = response.body();
// Gson이나 Jackson으로 파싱...</string>

User-agent: *
Disallow: /admin/
Disallow: /private/
Crawl-delay: 10

User-agent: Googlebot
Allow: /

news-crawler/
├── src/
│   ├── crawler/
│   │   ├── NewsCrawler.java
│   │   ├── CrawlerConfig.java
│   │   └── RateLimiter.java
│   ├── model/
│   │   └── Article.java
│   ├── parser/
│   │   ├── Parser.java
│   │   └── SiteParser.java
│   ├── storage/
│   │   ├── CsvStorage.java
│   │   └── DatabaseStorage.java
│   └── Main.java
└── pom.xml

package model;

import java.time.LocalDateTime;

public class Article {
    private String title;
    private String content;
    private String author;
    private String url;
    private String source;
    private LocalDateTime publishedDate;
    private LocalDateTime crawledDate;
    
    public Article() {
        this.crawledDate = LocalDateTime.now();
    }
    
    // Getters and Setters
    public String getTitle() { return title; }
    public void setTitle(String title) { this.title = title; }
    
    public String getContent() { return content; }
    public void setContent(String content) { this.content = content; }
    
    public String getAuthor() { return author; }
    public void setAuthor(String author) { this.author = author; }
    
    public String getUrl() { return url; }
    public void setUrl(String url) { this.url = url; }
    
    public String getSource() { return source; }
    public void setSource(String source) { this.source = source; }
    
    public LocalDateTime getPublishedDate() { return publishedDate; }
    public void setPublishedDate(LocalDateTime publishedDate) { 
        this.publishedDate = publishedDate; 
    }
    
    public LocalDateTime getCrawledDate() { return crawledDate; }
    
    @Override
    public String toString() {
        return String.format("[%s] %s - %s", source, title, author);
    }
}

package crawler;

public class CrawlerConfig {
    private int timeout = 5000;
    private int retryCount = 3;
    private int delayBetweenRequests = 2000;
    private String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36";
    
    public int getTimeout() { return timeout; }
    public void setTimeout(int timeout) { this.timeout = timeout; }
    
    public int getRetryCount() { return retryCount; }
    public void setRetryCount(int retryCount) { this.retryCount = retryCount; }
    
    public int getDelayBetweenRequests() { return delayBetweenRequests; }
    public void setDelayBetweenRequests(int delay) { 
        this.delayBetweenRequests = delay; 
    }
    
    public String getUserAgent() { return userAgent; }
    public void setUserAgent(String userAgent) { this.userAgent = userAgent; }
}

package parser;

import model.Article;
import org.jsoup.nodes.Document;
import java.util.List;

public interface SiteParser {
    String getSiteName();
    List<string> getArticleUrls(Document doc);
    Article parseArticle(Document doc);
}</string>

package parser;

import model.Article;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.util.ArrayList;
import java.util.List;

public class ExampleNewsParser implements SiteParser {
    
    @Override
    public String getSiteName() {
        return "Example News";
    }
    
    @Override
    public List<string> getArticleUrls(Document doc) {
        List<string> urls = new ArrayList<>();
        Elements links = doc.select("div.article-list a.article-link");
        
        for (Element link : links) {
            String url = link.absUrl("href");
            if (!url.isEmpty()) {
                urls.add(url);
            }
        }
        
        return urls;
    }
    
    @Override
    public Article parseArticle(Document doc) {
        Article article = new Article();
        article.setSource(getSiteName());
        
        // 제목 추출
        Element titleElement = doc.selectFirst("h1.article-title");
        if (titleElement != null) {
            article.setTitle(titleElement.text());
        }
        
        // 작성자 추출
        Element authorElement = doc.selectFirst("span.author-name");
        if (authorElement != null) {
            article.setAuthor(authorElement.text());
        }
        
        // 본문 추출
        Element contentElement = doc.selectFirst("div.article-content");
        if (contentElement != null) {
            article.setContent(contentElement.text());
        }
        
        // URL 설정
        article.setUrl(doc.location());
        
        return article;
    }
}</string></string>

package crawler;

import model.Article;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import parser.SiteParser;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;

public class NewsCrawler {
    private CrawlerConfig config;
    private SiteParser parser;
    
    public NewsCrawler(CrawlerConfig config, SiteParser parser) {
        this.config = config;
        this.parser = parser;
    }
    
    public List</code><article><code> crawl(String listPageUrl) {
        List</code><article><code> articles = new ArrayList<>();
        
        try {
            System.out.println("목록 페이지 크롤링: " + listPageUrl);
            Document listDoc = fetchDocument(listPageUrl);
            List<string> articleUrls = parser.getArticleUrls(listDoc);
            
            System.out.println(articleUrls.size() + "개의 기사 URL 발견");
            
            for (String url : articleUrls) {
                try {
                    System.out.println("기사 크롤링: " + url);
                    Document articleDoc = fetchDocument(url);
                    Article article = parser.parseArticle(articleDoc);
                    articles.add(article);
                    
                    // 딜레이
                    TimeUnit.MILLISECONDS.sleep(config.getDelayBetweenRequests());
                    
                } catch (Exception e) {
                    System.err.println("기사 크롤링 실패: " + url);
                    e.printStackTrace();
                }
            }
            
        } catch (Exception e) {
            System.err.println("목록 페이지 크롤링 실패");
            e.printStackTrace();
        }
        
        return articles;
    }
    
    private Document fetchDocument(String url) throws IOException {
        int attempts = 0;
        
        while (attempts < config.getRetryCount()) {
            try {
                return Jsoup.connect(url)
                        .userAgent(config.getUserAgent())
                        .timeout(config.getTimeout())
                        .get();
                        
            } catch (IOException e) {
                attempts++;
                if (attempts >= config.getRetryCount()) {
                    throw e;
                }
                
                try {
                    TimeUnit.SECONDS.sleep(2);
                } catch (InterruptedException ie) {
                    Thread.currentThread().interrupt();
                    throw new IOException("재시도 중 인터럽트", ie);
                }
            }
        }
        
        throw new IOException("문서 가져오기 실패");
    }
}</string></code></article></article></pre>
</div>

<h3>🎬 Main 클래스</h3>

<div class="tx-code-block">
<pre><code>import crawler.CrawlerConfig;
import crawler.NewsCrawler;
import model.Article;
import parser.ExampleNewsParser;
import storage.CsvStorage;
import java.util.List;

public class Main {
    public static void main(String[] args) {
        System.out.println("=== 뉴스 크롤러 시작 ===\n");
        
        // 설정
        CrawlerConfig config = new CrawlerConfig();
        config.setTimeout(5000);
        config.setDelayBetweenRequests(2000);
        
        // 파서 설정
        ExampleNewsParser parser = new ExampleNewsParser();
        
        // 크롤러 생성
        NewsCrawler crawler = new NewsCrawler(config, parser);
        
        // 크롤링 실행
        String targetUrl = "https://example-news.com/latest";
        List</code><article><code> articles = crawler.crawl(targetUrl);
        
        // 결과 출력
        System.out.println("\n=== 크롤링 완료 ===");
        System.out.println("총 " + articles.size() + "개의 기사 수집");
        
        for (Article article : articles) {
            System.out.println(article);
        }
        
        // CSV로 저장
        CsvStorage storage = new CsvStorage();
        storage.save(articles, "news_articles.csv");
        
        System.out.println("\n프로그램 종료");
    }
}</code></article></pre>
</div>

<p>
와! 꽤 본격적인 프로젝트가 됐지? 😎 이 구조는 확장하기도 쉬워. 새로운 뉴스 사이트를 추가하려면 SiteParser 인터페이스를 구현하는 새 클래스만 만들면 돼!
</p>

<hr class="tx-divider">

<h2>🐛 디버깅과 문제 해결</h2>

<p>
크롤링하다 보면 여러 문제를 만나게 돼. 자주 발생하는 문제들과 해결 방법을 알아보자. 🔧
</p>

<h3>❌ 자주 만나는 에러들</h3>

<p>
<strong>1. HTTP 403 Forbidden</strong><br>
서버가 접근을 거부하는 거야. User-Agent를 설정하지 않았거나, 봇으로 인식됐을 가능성이 높아.
</p>

<div class="tx-code-block">
<pre><code>// 해결 방법
Document doc = Jsoup.connect(url)
    .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
    .referrer("https://www.google.com") // Referrer도 추가
    .get();

// 타임아웃 10초로 증가
Document doc = Jsoup.connect(url)
    .timeout(10000)
    .get();

// 방어적 코딩
Element titleElement = doc.selectFirst("h1.title");
if (titleElement != null) {
    String title = titleElement.text();
} else {
    System.err.println("제목을 찾을 수 없습니다.");
    // 대체 셀렉터 시도
    titleElement = doc.selectFirst("h1");
}

Document doc = Jsoup.connect(url)
    .userAgent("Mozilla/5.0")
    .get();

// 또는 HTML 문자열을 파싱할 때
Document doc = Jsoup.parse(html, "UTF-8");

// 전체 HTML 출력
System.out.println(doc.html());

// 특정 요소만 출력
Element content = doc.selectFirst("div.content");
if (content != null) {
    System.out.println(content.html());
}

import java.util.logging.*;

Logger logger = Logger.getLogger(NewsCrawler.class.getName());

logger.info("크롤링 시작: " + url);
logger.warning("요소를 찾을 수 없음: " + selector);
logger.severe("크롤링 실패: " + e.getMessage());

import java.nio.file.*;

String html = doc.html();
Files.writeString(
    Paths.get("debug_page.html"), 
    html, 
    StandardCharsets.UTF_8
);

import java.util.concurrent.*;
import java.util.List;
import java.util.ArrayList;

public class ParallelCrawler {
    private static final int THREAD_POOL_SIZE = 5;
    
    public List</code><article><code> crawlParallel(List<string> urls) {
        ExecutorService executor = Executors.newFixedThreadPool(THREAD_POOL_SIZE);
        List<future<article>> futures = new ArrayList<>();
        
        // 각 URL을 별도 스레드에서 처리
        for (String url : urls) {
            Future</future<article></string></code><article><code> future = executor.submit(() -> {
                try {
                    Document doc = Jsoup.connect(url)
                            .userAgent("Mozilla/5.0")
                            .timeout(5000)
                            .get();
                    return parseArticle(doc);
                } catch (Exception e) {
                    System.err.println("크롤링 실패: " + url);
                    return null;
                }
            });
            futures.add(future);
        }
        
        // 결과 수집
        List</code><article><code> articles = new ArrayList<>();
        for (Future</code><article><code> future : futures) {
            try {
                Article article = future.get(10, TimeUnit.SECONDS);
                if (article != null) {
                    articles.add(article);
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        
        executor.shutdown();
        return articles;
    }
    
    private Article parseArticle(Document doc) {
        // 파싱 로직
        return new Article();
    }
}</code></article></article></article></article></pre>
</div>

<div class="tx-warning">
<strong>⚠️ 주의:</strong> 멀티스레딩을 사용할 때는 서버에 과부하를 주지 않도록 스레드 수를 적절히 제한해야 해. 보통 5~10개 정도가 적당해. 그리고 각 스레드에서도 딜레이를 넣는 걸 잊지 마!
</div>

<h3>💾 캐싱 활용하기</h3>

<p>
같은 페이지를 여러 번 요청하지 않도록 캐싱을 구현하자.
</p>

<div class="tx-code-block">
<pre><code>import java.util.concurrent.ConcurrentHashMap;
import java.util.Map;

public class CachedCrawler {
    private Map<string, document=""> cache = new ConcurrentHashMap<>();
    
    public Document fetchWithCache(String url) throws IOException {
        // 캐시에 있으면 반환
        if (cache.containsKey(url)) {
            System.out.println("캐시에서 로드: " + url);
            return cache.get(url);
        }
        
        // 없으면 크롤링
        System.out.println("새로 크롤링: " + url);
        Document doc = Jsoup.connect(url)
                .userAgent("Mozilla/5.0")
                .timeout(5000)
                .get();
        
        // 캐시에 저장
        cache.put(url, doc);
        return doc;
    }
    
    public void clearCache() {
        cache.clear();
    }
}</string,>

public class ProgressTracker {
    private int total;
    private int completed;
    private int failed;
    
    public ProgressTracker(int total) {
        this.total = total;
        this.completed = 0;
        this.failed = 0;
    }
    
    public synchronized void incrementCompleted() {
        completed++;
        printProgress();
    }
    
    public synchronized void incrementFailed() {
        failed++;
        printProgress();
    }
    
    private void printProgress() {
        int percentage = (int) ((completed + failed) * 100.0 / total);
        System.out.printf("\r진행률: %d%% (%d/%d 완료, %d 실패)", 
                percentage, completed, total, failed);
    }
}

import java.net.Proxy;
import java.net.InetSocketAddress;

public class ProxyCrawler {
    public Document fetchWithProxy(String url, String proxyHost, int proxyPort) 
            throws IOException {
        
        Proxy proxy = new Proxy(
            Proxy.Type.HTTP, 
            new InetSocketAddress(proxyHost, proxyPort)
        );
        
        Document doc = Jsoup.connect(url)
                .proxy(proxy)
                .userAgent("Mozilla/5.0")
                .timeout(10000)
                .get();
        
        return doc;
    }
}

import java.util.Random;

public class UserAgentRotator {
    private static final String[] USER_AGENTS = {
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
        "Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15",
        "Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15"
    };
    
    private Random random = new Random();
    
    public String getRandomUserAgent() {
        return USER_AGENTS[random.nextInt(USER_AGENTS.length)];
    }
    
    public Document fetch(String url) throws IOException {
        return Jsoup.connect(url)
                .userAgent(getRandomUserAgent())
                .timeout(5000)
                .get();
    }
}

import java.util.regex.*;

public class DataCleaner {
    
    // 여러 공백을 하나로
    public static String normalizeWhitespace(String text) {
        return text.replaceAll("\\s+", " ").trim();
    }
    
    // HTML 태그 제거
    public static String removeHtmlTags(String text) {
        return text.replaceAll("<[^>]*>", "");
    }
    
    // 이메일 추출
    public static List<string> extractEmails(String text) {
        List<string> emails = new ArrayList<>();
        Pattern pattern = Pattern.compile(
            "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"
        );
        Matcher matcher = pattern.matcher(text);
        
        while (matcher.find()) {
            emails.add(matcher.group());
        }
        
        return emails;
    }
    
    // 전화번호 추출 (한국)
    public static List<string> extractPhoneNumbers(String text) {
        List<string> phones = new ArrayList<>();
        Pattern pattern = Pattern.compile(
            "0\\d{1,2}-\\d{3,4}-\\d{4}"
        );
        Matcher matcher = pattern.matcher(text);
        
        while (matcher.find()) {
            phones.add(matcher.group());
        }
        
        return phones;
    }
    
    // 가격 추출
    public static List<integer> extractPrices(String text) {
        List<integer> prices = new ArrayList<>();
        Pattern pattern = Pattern.compile(
            "(\\d{1,3}(,\\d{3})*|\\d+)원"
        );
        Matcher matcher = pattern.matcher(text);
        
        while (matcher.find()) {
            String priceStr = matcher.group(1).replace(",", "");
            prices.add(Integer.parseInt(priceStr));
        }
        
        return prices;
    }
}</integer></integer></string></string></string></string>

import java.time.LocalDateTime;
import java.util.Set;
import java.util.HashSet;

public class IncrementalCrawler {
    private Set<string> crawledUrls = new HashSet<>();
    private LocalDateTime lastCrawlTime;
    
    public List</string></code><article><code> crawlNew(String listPageUrl) {
        List</code><article><code> newArticles = new ArrayList<>();
        
        try {
            Document doc = Jsoup.connect(listPageUrl).get();
            Elements articleLinks = doc.select("a.article-link");
            
            for (Element link : articleLinks) {
                String url = link.absUrl("href");
                
                // 이미 크롤링한 URL은 건너뛰기
                if (crawledUrls.contains(url)) {
                    System.out.println("이미 크롤링함: " + url);
                    continue;
                }
                
                // 새 기사 크롤링
                Document articleDoc = Jsoup.connect(url).get();
                Article article = parseArticle(articleDoc);
                
                // 마지막 크롤링 이후의 기사만
                if (lastCrawlTime == null || 
                    article.getPublishedDate().isAfter(lastCrawlTime)) {
                    newArticles.add(article);
                    crawledUrls.add(url);
                }
            }
            
            lastCrawlTime = LocalDateTime.now();
            
        } catch (Exception e) {
            e.printStackTrace();
        }
        
        return newArticles;
    }
    
    private Article parseArticle(Document doc) {
        // 파싱 로직
        return new Article();
    }
    
    public void saveCrawledUrls(String filename) throws IOException {
        Files.write(
            Paths.get(filename), 
            crawledUrls, 
            StandardCharsets.UTF_8
        );
    }
    
    public void loadCrawledUrls(String filename) throws IOException {
        if (Files.exists(Paths.get(filename))) {
            crawledUrls = new HashSet<>(
                Files.readAllLines(Paths.get(filename), StandardCharsets.UTF_8)
            );
        }
    }
}</code></article></article></pre>
</div>

<hr class="tx-divider">

<h2>🌐 실제 사용 사례</h2>

<p>
이론만 배우면 재미없잖아? 실제로 어떻게 활용되는지 몇 가지 사례를 보자. 💼
</p>

<h3>💰 가격 비교 서비스</h3>

<p>
여러 쇼핑몰의 상품 가격을 수집해서 최저가를 찾아주는 서비스야. 실제로 많은 가격 비교 사이트들이 이런 방식으로 운영돼.
</p>

<div class="tx-code-block">
<pre><code>public class PriceComparison {
    public List<productprice> comparePrice(String productName) {
        List<productprice> prices = new ArrayList<>();
        
        // 여러 쇼핑몰 크롤링
        prices.addAll(crawlSiteA(productName));
        prices.addAll(crawlSiteB(productName));
        prices.addAll(crawlSiteC(productName));
        
        // 가격순 정렬
        prices.sort(Comparator.comparing(ProductPrice::getPrice));
        
        return prices;
    }
    
    private List<productprice> crawlSiteA(String productName) {
        // 사이트 A 크롤링 로직
        return new ArrayList<>();
    }
}

class ProductPrice {
    private String siteName;
    private String productName;
    private int price;
    private String url;
    
    // getters, setters...
}</productprice></productprice></productprice>

public class RealEstateCrawler {
    public List<property> crawlProperties(String region) {
        List<property> properties = new ArrayList<>();
        
        try {
            String url = "https://realestate-site.com/search?region=" + region;
            Document doc = Jsoup.connect(url).get();
            
            Elements listings = doc.select("div.property-listing");
            
            for (Element listing : listings) {
                Property property = new Property();
                property.setRegion(region);
                property.setType(listing.select(".property-type").text());
                property.setPrice(parsePrice(listing.select(".price").text()));
                property.setArea(parseArea(listing.select(".area").text()));
                property.setFloor(listing.select(".floor").text());
                
                properties.add(property);
            }
            
        } catch (Exception e) {
            e.printStackTrace();
        }
        
        return properties;
    }
    
    private int parsePrice(String priceText) {
        // "3억 5천만원" -> 350000000
        // 파싱 로직...
        return 0;
    }
    
    private double parseArea(String areaText) {
        // "85㎡" -> 85.0
        return 0.0;
    }
}</property></property>

import java.util.Timer;
import java.util.TimerTask;

public class NewsMonitorBot {
    private Set notifiedArticles = new HashSet<>();
    private List keywords;
    
    public NewsMonitorBot(List keywords) {
        this.keywords = keywords;
    }
    
    public void startMonitoring(int intervalMinutes) {
        Timer timer = new Timer();
        
        timer.scheduleAtFixedRate(new TimerTask() {
            @Override
            public void run() {
                checkForNewArticles();
            }
        }, 0, intervalMinutes * 60 * 1000);
    }
    
    private void checkForNewArticles() {
        try {
            List
 articles = crawlNews();
            
            for (Article article : articles) {
                if (notifiedArticles.contains(article.getUrl())) {
                    continue;
                }
                
                // 키워드 포함 여부 확인
                for (String keyword : keywords) {
                    if (article.getTitle().contains(keyword) || 
                        article.getContent().contains(keyword)) {
                        
                        sendNotification(article, keyword);
                        notifiedArticles.add(article.getUrl());
                        break;
                    }
                }
            }
            
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    
    private List
 crawlNews() {
        // 뉴스 크롤링 로직
        return new ArrayList<>();
    }
    
    private void sendNotification(Article article, String keyword) {
        System.out.println("🔔 키워드 '" + keyword + "' 발견!");
        System.out.println("제목: " + article.getTitle());
        System.out.println("URL: " + article.getUrl());
        
        // 이메일, 슬랙, 텔레그램 등으로 알림 전송
    }
}