import csv from vector_db import VectorDB from embedding_client import EmbeddingClient from config import EMBDED_MODEL_NAME import os from semantic_chunker import chunk_text_semantic, add_header_to_chunk CSV_PATH = "datasets/final.csv" def load_csv_texts_with_meta(path): data = [] with open(path, "r", encoding="utf-8-sig") as f: reader = csv.DictReader(f) for row in reader: if all(k in row for k in ["instruction", "response", "category", "intent", "flags"]): instruction = row['instruction'].strip() response = row['response'].strip() category = row['category'].strip() intent = row['intent'].strip() flags = row['flags'].strip() combined_text = f"Soru: {instruction}\nCevap: {response}" chunks = chunk_text_semantic(combined_text, max_tokens=256) for chunk in chunks: chunk_with_header = add_header_to_chunk(category, chunk) # ya da intent data.append({ "text": chunk_with_header, "category": category, "intent": intent, "flags": flags }) return data def build_vector_db_from_csv_with_meta(csv_path): data = load_csv_texts_with_meta(csv_path) print(f"Toplam {len(data)} kayıt bulundu.") embedder = EmbeddingClient() vector_db = VectorDB() for i, entry in enumerate(data): text = entry["text"] vec = embedder.get_embedding(text) # Eğer vector_db.add_with_text fonksiyonu metadata kabul ediyorsa: vector_db.add_with_text(vec, text, metadata={ "category": entry["category"], "intent": entry["intent"], "flags": entry["flags"] }) if i % 100 == 0: vector_db.save() print(f"{i}/{len(data)} eklendi...") vector_db.save() print(f"{len(data)} metin vektör DB'ye eklendi ve kaydedildi.") if __name__ == "__main__": if not os.path.exists(CSV_PATH): print(f"CSV dosyası bulunamadı: {CSV_PATH}") else: build_vector_db_from_csv_with_meta(CSV_PATH)