feat: regulation KB 데이터 보강 — expert_forecasts_ko + KMAC/NH 리포트 인덱싱

- build_expert_forecasts_kb.py: expert_forecasts.json → expert_forecasts_ko (8청크)
- 기관별 가격 전망, 시장 트렌드, 지정학 요인 문서화
- build_market_reports_kb.py: KMAC/NH PDF OCR → chroma_db_regulation (691→728청크)
- CHROMA_PATH: chroma_db_methodology → chroma_db_regulation 변경
- NH투자증권, KMAC 월간 리포트 6개 (9~3월호) 추가
- law/strategy specialist: search_methodology_knowledge_base 추가 (이전 커밋)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

build_expert_forecasts_kb.py

@@ -0,0 +1,202 @@
+"""
+expert_forecasts.json → ChromaDB expert_forecasts_ko 컬렉션 인덱싱
+
+실행:
+  cd carbon-ai-chatbot
+  python build_expert_forecasts_kb.py [--dry-run]
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import logging
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).parent
+sys.path.insert(0, str(ROOT / "react-agent" / "src"))
+
+from dotenv import load_dotenv
+load_dotenv(ROOT / "react-agent" / ".env")
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+logger = logging.getLogger(__name__)
+
+SOURCE_FILE = ROOT / "kau_market_reports" / "expert_forecasts.json"
+CHROMA_PATH = ROOT / "react-agent" / "chroma_db_regulation"
+COLLECTION  = "expert_forecasts_ko"
+EMBED_MODEL = "nlpai-lab/KURE-v1"
+
+
+def build_documents(data: dict) -> list[dict]:
+    """expert_forecasts.json → 검색 최적화 문서 청크 목록"""
+    docs = []
+
+    # ── 1. 기관별 가격 전망 ──────────────────────────────────────────────
+    for fc in data.get("forecasts", []):
+        inst   = fc.get("institution", "")
+        year   = fc.get("year", "")
+        lo     = fc.get("forecast_low", 0)
+        mid    = fc.get("forecast_mid", 0)
+        hi     = fc.get("forecast_high", 0)
+        date   = fc.get("collected_date", "")
+        drivers = "\n".join(f"- {d}" for d in fc.get("key_drivers", []))
+        content = (
+            f"[기관 KAU 가격 전망] {inst} ({year}년)\n"
+            f"전망 범위: {lo:,}원 ~ {hi:,}원 / 중간값: {mid:,}원\n"
+            f"주요 근거:\n{drivers}\n"
+            f"수집일: {date}"
+        )
+        docs.append({
+            "content": content,
+            "metadata": {
+                "source": "expert_forecasts.json",
+                "doc_type": "price_forecast",
+                "institution": inst,
+                "year": str(year),
+                "forecast_low": lo,
+                "forecast_mid": mid,
+                "forecast_high": hi,
+                "collected_date": date,
+            }
+        })
+
+    # ── 2. 시장 트렌드 분석 ──────────────────────────────────────────────
+    trend = data.get("trend", {})
+    if trend:
+        bull = "\n".join(f"- {f}" for f in trend.get("bullish_factors", []))
+        bear = "\n".join(f"- {f}" for f in trend.get("bearish_factors", []))
+        events = "\n".join(f"- {e}" for e in trend.get("key_events", []))
+        summary = trend.get("trend_summary", "")
+        content = (
+            f"[KAU 시장 트렌드 분석]\n"
+            f"{summary}\n\n"
+            f"## 상승 요인\n{bull}\n\n"
+            f"## 하락 요인\n{bear}\n\n"
+            f"## 주요 이벤트\n{events}"
+        )
+        docs.append({
+            "content": content,
+            "metadata": {
+                "source": "expert_forecasts.json",
+                "doc_type": "market_trend",
+                "institution": "종합",
+                "confidence": trend.get("confidence", 0),
+                "collected_date": trend.get("collected_date", ""),
+            }
+        })
+
+    # ── 3. 지정학·국제 요인 ──────────────────────────────────────────────
+    geo = data.get("geopolitical", {})
+    for factor in geo.get("international_factors", []):
+        event   = factor.get("event", "")
+        region  = factor.get("region", "")
+        impact  = factor.get("impact_summary", "")
+        direction = factor.get("impact_direction", "")
+        timeframe = factor.get("timeframe", "")
+        content = (
+            f"[국제 탄소시장 지정학 요인] {region}\n"
+            f"이벤트: {event}\n"
+            f"영향 방향: {direction} ({timeframe})\n"
+            f"영향 분석: {impact}"
+        )
+        docs.append({
+            "content": content,
+            "metadata": {
+                "source": "expert_forecasts.json",
+                "doc_type": "geopolitical_factor",
+                "region": region,
+                "direction": direction,
+                "timeframe": timeframe,
+            }
+        })
+
+    # ── 4. 국내 정책 요인 ────────────────────────────────────────────────
+    for factor in geo.get("domestic_policy_factors", []):
+        policy  = factor.get("policy", "")
+        impact  = factor.get("impact_summary", "")
+        direction = factor.get("impact_direction", "")
+        timeframe = factor.get("timeframe", "")
+        content = (
+            f"[K-ETS 국내 정책 요인]\n"
+            f"정책: {policy}\n"
+            f"영향 방향: {direction} ({timeframe})\n"
+            f"영향 분석: {impact}"
+        )
+        docs.append({
+            "content": content,
+            "metadata": {
+                "source": "expert_forecasts.json",
+                "doc_type": "domestic_policy",
+                "direction": direction,
+                "timeframe": timeframe,
+            }
+        })
+
+    return [d for d in docs if len(d["content"].strip()) > 30]
+
+
+def index_documents(docs: list[dict], dry_run: bool = False):
+    if dry_run:
+        logger.info(f"[DRY-RUN] {len(docs)}개 문서 스킵")
+        for d in docs[:3]:
+            logger.info(f"  - {d['metadata']['doc_type']}: {d['content'][:80]}...")
+        return
+
+    from langchain_chroma import Chroma
+    from langchain_core.documents import Document
+    from langchain_huggingface import HuggingFaceEmbeddings
+
+    embeddings = HuggingFaceEmbeddings(
+        model_name=EMBED_MODEL,
+        model_kwargs={"device": "cpu"},
+        encode_kwargs={"normalize_embeddings": True},
+    )
+
+    import chromadb
+    client = chromadb.PersistentClient(path=str(CHROMA_PATH))
+
+    # 기존 컬렉션 삭제 후 재생성
+    try:
+        client.delete_collection(COLLECTION)
+        logger.info(f"기존 '{COLLECTION}' 컬렉션 삭제")
+    except Exception:
+        pass
+
+    lc_docs, ids = [], []
+    for i, d in enumerate(docs):
+        lc_docs.append(Document(page_content=d["content"], metadata=d["metadata"]))
+        raw_id = f"expert_forecast_{i}_{d['metadata']['doc_type']}"
+        ids.append(hashlib.md5(raw_id.encode()).hexdigest())
+
+    vs = Chroma(
+        collection_name=COLLECTION,
+        embedding_function=embeddings,
+        persist_directory=str(CHROMA_PATH),
+    )
+    vs.add_documents(lc_docs, ids=ids)
+    logger.info(f"✅ {len(lc_docs)}청크 → '{COLLECTION}' 저장 완료")
+
+
+def main(dry_run: bool = False):
+    if not SOURCE_FILE.exists():
+        logger.error(f"파일 없음: {SOURCE_FILE}")
+        return
+
+    data = json.loads(SOURCE_FILE.read_text(encoding="utf-8"))
+    docs = build_documents(data)
+    logger.info(f"생성된 문서 수: {len(docs)}")
+
+    for d in docs:
+        logger.info(f"  [{d['metadata']['doc_type']}] {d['content'][:60].strip()}")
+
+    index_documents(docs, dry_run=dry_run)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+    main(dry_run=args.dry_run)

build_market_reports_kb.py

@@ -0,0 +1,272 @@
+"""
+kau_market_reports/ OCR + ChromaDB 인덱싱 파이프라인
+
+처리 흐름:
+  1. kau_market_reports/*.pdf  (스캔 PDF → Gemini OCR)
+  2. 텍스트 청크 분할
+  3. ChromaDB kau_market_reports 컬렉션 저장 (KURE-v1 임베딩)
+  4. 파라미터 추출 파일 저장 (seasonal_bias 보정용)
+
+실행:
+  cd carbon-ai-chatbot
+  python build_market_reports_kb.py [--dry-run]
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import hashlib
+import json
+import logging
+import os
+import re
+import sys
+from datetime import datetime
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+# 환경변수 로드
+ROOT = Path(__file__).parent
+load_dotenv(ROOT / "react-agent" / ".env")
+
+# GOOGLE_API_KEY 보정 (GEMINI_API_KEY → GOOGLE_API_KEY fallback)
+if not os.environ.get("GOOGLE_API_KEY") and os.environ.get("GEMINI_API_KEY"):
+    os.environ["GOOGLE_API_KEY"] = os.environ["GEMINI_API_KEY"]
+
+sys.path.insert(0, str(ROOT / "react-agent" / "src"))
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+logger = logging.getLogger(__name__)
+
+# ── 상수 ───────────────────────────────────────────────────────────────────
+SOURCE_DIR    = ROOT / "kau_market_reports"
+CHROMA_PATH   = ROOT / "react-agent" / "chroma_db_regulation"
+COLLECTION    = "kau_market_reports"
+EMBED_MODEL   = "nlpai-lab/KURE-v1"
+CHUNK_SIZE    = 900
+CHUNK_OVERLAP = 150
+PROGRESS_FILE = SOURCE_DIR / ".ocr_progress.json"
+PARAMS_OUTPUT = SOURCE_DIR / "extracted_params.json"
+
+
+# ── 진행 상태 관리 ─────────────────────────────────────────────────────────
+
+def load_progress() -> dict:
+    if PROGRESS_FILE.exists():
+        return json.loads(PROGRESS_FILE.read_text(encoding="utf-8"))
+    return {}
+
+
+def save_progress(progress: dict):
+    PROGRESS_FILE.write_text(json.dumps(progress, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+# ── 텍스트 청크 분할 ────────────────────────────────────────────────────────
+
+def smart_chunk(text: str, chunk_size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP) -> list[str]:
+    """문단 경계 우선 청크 분할"""
+    paragraphs = re.split(r"\n{2,}", text.strip())
+    chunks, current = [], ""
+    for para in paragraphs:
+        para = para.strip()
+        if not para:
+            continue
+        if len(current) + len(para) + 2 <= chunk_size:
+            current = (current + "\n\n" + para).strip()
+        else:
+            if current:
+                chunks.append(current)
+            if len(para) > chunk_size:
+                # 긴 단락은 강제 분할
+                for i in range(0, len(para), chunk_size - overlap):
+                    chunks.append(para[i:i + chunk_size])
+            else:
+                current = para
+    if current:
+        chunks.append(current)
+    return [c for c in chunks if len(c.strip()) > 50]
+
+
+# ── OCR (Gemini) ────────────────────────────────────────────────────────────
+
+async def ocr_pdf(pdf_path: Path, max_pages: int = 30) -> str:
+    """스캔 PDF → Gemini OCR → 전체 텍스트"""
+    from react_agent.ocr_tool import extract_text_from_pdf
+    logger.info(f"[OCR] {pdf_path.name} 처리 중...")
+    text = await extract_text_from_pdf(str(pdf_path), max_pages=max_pages)
+    logger.info(f"[OCR] {pdf_path.name} 완료: {len(text)}자")
+    return text
+
+
+# ── 파라미터 추출 ───────────────────────────────────────────────────────────
+
+def extract_parameters(filename: str, text: str) -> dict:
+    """OCR 텍스트에서 수치 파라미터 추출 (향후 seasonal_bias 보정용)"""
+    params = {"source": filename, "raw_snippets": []}
+
+    # 월별 수익률/상승/하락 패턴 검출
+    month_patterns = [
+        r"(\d{1,2})월[^\n]{0,30}([+-]?\d+\.?\d*)%",
+        r"([+-]?\d+\.?\d*)%[^\n]{0,20}(\d{1,2})월",
+    ]
+    for pat in month_patterns:
+        for m in re.finditer(pat, text):
+            params["raw_snippets"].append(m.group(0).strip())
+
+    # 가격 목표치/전망 패턴
+    price_patterns = [
+        r"(목표가|적정가|전망가|예상가)[^\n]{0,20}(\d{4,6})[~\-～]?(\d{4,6})?원",
+        r"KAU\d{2}[^\n]{0,30}(\d{4,6})[~\-～](\d{4,6})원",
+    ]
+    for pat in price_patterns:
+        for m in re.finditer(pat, text):
+            params["raw_snippets"].append(m.group(0).strip())
+
+    # 정산/명세서 관련 언급
+    event_patterns = [
+        r"(정산|명세서|할당)[^\n]{0,60}([+-]?\d+\.?\d*)%",
+    ]
+    for pat in event_patterns:
+        for m in re.finditer(pat, text):
+            params["raw_snippets"].append(m.group(0).strip())
+
+    params["snippet_count"] = len(params["raw_snippets"])
+    return params
+
+
+# ── ChromaDB 인덱싱 ─────────────────────────────────────────────────────────
+
+def embed_chunks(chunks_data: list[dict], dry_run: bool = False):
+    if dry_run:
+        logger.info(f"[DRY-RUN] {len(chunks_data)}청크 스킵")
+        return
+
+    try:
+        from langchain_chroma import Chroma
+        from langchain_core.documents import Document
+        from langchain_huggingface import HuggingFaceEmbeddings
+
+        embeddings = HuggingFaceEmbeddings(
+            model_name=EMBED_MODEL,
+            model_kwargs={"device": "cpu"},
+            encode_kwargs={"normalize_embeddings": True},
+        )
+
+        docs, ids = [], []
+        for c in chunks_data:
+            doc = Document(page_content=c["content"], metadata=c["metadata"])
+            raw_id = f"{c['metadata']['source']}_{c['metadata']['chunk_index']}"
+            docs.append(doc)
+            ids.append(hashlib.md5(raw_id.encode()).hexdigest())
+
+        vs = Chroma(
+            collection_name=COLLECTION,
+            embedding_function=embeddings,
+            persist_directory=str(CHROMA_PATH),
+        )
+        vs.add_documents(docs, ids=ids)
+        logger.info(f"✅ {len(docs)}청크 → ChromaDB '{COLLECTION}' 저장 완료")
+
+    except Exception as e:
+        logger.error(f"임베딩 실패: {e}")
+        raise
+
+
+# ── 메인 파이프라인 ─────────────────────────────────────────────────────────
+
+async def process_file(pdf_path: Path, progress: dict, dry_run: bool) -> dict | None:
+    """단일 PDF OCR + 청크 + 임베딩"""
+    fname = pdf_path.name
+
+    if fname in progress and progress[fname].get("done"):
+        logger.info(f"[스킵] {fname} (이미 처리됨)")
+        return None
+
+    # ① OCR
+    text = await ocr_pdf(pdf_path)
+    if not text or text.startswith("[오류]") or text == "[추출된 텍스트 없음]":
+        logger.warning(f"[OCR 실패] {fname}: {text[:100]}")
+        return None
+
+    # ② 메타데이터 추출 (파일명에서 기관/월 파싱)
+    institution = "KMAC" if "KMAC" in fname else ("NH투자증권" if "NH" in fname else "기타")
+    report_month = ""
+    m = re.search(r"(\d{1,2})월", fname)
+    if m:
+        report_month = f"{m.group(1)}월"
+    m2 = re.search(r"(\d{2})년", fname)
+    report_year = f"20{m2.group(1)}" if m2 else "2025"
+
+    # ③ 청크 분할
+    chunks = smart_chunk(text)
+    chunks_data = [
+        {
+            "content": chunk,
+            "metadata": {
+                "source":       fname,
+                "institution":  institution,
+                "report_month": report_month,
+                "report_year":  report_year,
+                "collection":   COLLECTION,
+                "chunk_index":  i,
+                "ocr_date":     datetime.now().strftime("%Y-%m-%d"),
+                "doc_type":     "market_analysis_report",
+            },
+        }
+        for i, chunk in enumerate(chunks)
+    ]
+
+    # ④ 임베딩
+    embed_chunks(chunks_data, dry_run=dry_run)
+
+    # ⑤ 파라미터 추출
+    params = extract_parameters(fname, text)
+
+    # ⑥ 진행 상태 저장
+    progress[fname] = {
+        "done":          True,
+        "chunks":        len(chunks),
+        "chars":         len(text),
+        "snippets":      params["snippet_count"],
+        "processed_at":  datetime.now().isoformat(),
+    }
+
+    logger.info(f"[완료] {fname}: {len(chunks)}청크, {params['snippet_count']}개 수치 스니펫")
+    return params
+
+
+async def main(dry_run: bool = False):
+    pdfs = sorted(SOURCE_DIR.glob("*.pdf"))
+    if not pdfs:
+        logger.error(f"PDF 없음: {SOURCE_DIR}")
+        return
+
+    logger.info(f"처리 대상: {len(pdfs)}개 PDF")
+    progress = load_progress()
+    all_params = []
+
+    for pdf in pdfs:
+        params = await process_file(pdf, progress, dry_run)
+        if params:
+            all_params.append(params)
+        save_progress(progress)
+
+    # 파라미터 추출 결과 저장
+    if all_params and not dry_run:
+        PARAMS_OUTPUT.write_text(
+            json.dumps(all_params, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        logger.info(f"📊 파라미터 추출 결과 저장: {PARAMS_OUTPUT}")
+
+    logger.info("=" * 60)
+    logger.info(f"완료: {sum(1 for v in progress.values() if v.get('done'))}개 처리됨")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dry-run", action="store_true", help="임베딩 스킵 (OCR만 테스트)")
+    args = parser.parse_args()
+    asyncio.run(main(dry_run=args.dry_run))