asistan / new_csv_loader.py
mericberktas's picture
Upload folder using huggingface_hub
ae9c08b verified
Raw History Blame Contribute Delete
2.31 kB
import csv
from vector_db import VectorDB
from embedding_client import EmbeddingClient
from config import EMBDED_MODEL_NAME
import os
from semantic_chunker import chunk_text_semantic, add_header_to_chunk
CSV_PATH = "datasets/final.csv"
def load_csv_texts_with_meta(path):
data = []
with open(path, "r", encoding="utf-8-sig") as f:
reader = csv.DictReader(f)
for row in reader:
if all(k in row for k in ["instruction", "response", "category", "intent", "flags"]):
instruction = row['instruction'].strip()
response = row['response'].strip()
category = row['category'].strip()
intent = row['intent'].strip()
flags = row['flags'].strip()
combined_text = f"Soru: {instruction}\nCevap: {response}"
chunks = chunk_text_semantic(combined_text, max_tokens=256)
for chunk in chunks:
chunk_with_header = add_header_to_chunk(category, chunk) # ya da intent
data.append({
"text": chunk_with_header,
"category": category,
"intent": intent,
"flags": flags
})
return data
def build_vector_db_from_csv_with_meta(csv_path):
data = load_csv_texts_with_meta(csv_path)
print(f"Toplam {len(data)} kayıt bulundu.")
embedder = EmbeddingClient()
vector_db = VectorDB()
for i, entry in enumerate(data):
text = entry["text"]
vec = embedder.get_embedding(text)
# Eğer vector_db.add_with_text fonksiyonu metadata kabul ediyorsa:
vector_db.add_with_text(vec, text, metadata={
"category": entry["category"],
"intent": entry["intent"],
"flags": entry["flags"]
})
if i % 100 == 0:
vector_db.save()
print(f"{i}/{len(data)} eklendi...")
vector_db.save()
print(f"{len(data)} metin vektör DB'ye eklendi ve kaydedildi.")
if __name__ == "__main__":
if not os.path.exists(CSV_PATH):
print(f"CSV dosyası bulunamadı: {CSV_PATH}")
else:
build_vector_db_from_csv_with_meta(CSV_PATH)