Spaces:
Runtime error
Runtime error
Download new_csv_loader.py from mericberktas/asistan: direct link, hf CLI and curl.
- Browser
- Download file 2.31 kB
-
https://huggingface.co/spaces/mericberktas/asistan/resolve/main/new_csv_loader.py
- Command line
-
hf download hf://spaces/mericberktas/asistan/new_csv_loader.py
-
curl -L -o new_csv_loader.py https://huggingface.co/spaces/mericberktas/asistan/resolve/main/new_csv_loader.py
2.31 kB
| import csv | |
| from vector_db import VectorDB | |
| from embedding_client import EmbeddingClient | |
| from config import EMBDED_MODEL_NAME | |
| import os | |
| from semantic_chunker import chunk_text_semantic, add_header_to_chunk | |
| CSV_PATH = "datasets/final.csv" | |
| def load_csv_texts_with_meta(path): | |
| data = [] | |
| with open(path, "r", encoding="utf-8-sig") as f: | |
| reader = csv.DictReader(f) | |
| for row in reader: | |
| if all(k in row for k in ["instruction", "response", "category", "intent", "flags"]): | |
| instruction = row['instruction'].strip() | |
| response = row['response'].strip() | |
| category = row['category'].strip() | |
| intent = row['intent'].strip() | |
| flags = row['flags'].strip() | |
| combined_text = f"Soru: {instruction}\nCevap: {response}" | |
| chunks = chunk_text_semantic(combined_text, max_tokens=256) | |
| for chunk in chunks: | |
| chunk_with_header = add_header_to_chunk(category, chunk) # ya da intent | |
| data.append({ | |
| "text": chunk_with_header, | |
| "category": category, | |
| "intent": intent, | |
| "flags": flags | |
| }) | |
| return data | |
| def build_vector_db_from_csv_with_meta(csv_path): | |
| data = load_csv_texts_with_meta(csv_path) | |
| print(f"Toplam {len(data)} kayıt bulundu.") | |
| embedder = EmbeddingClient() | |
| vector_db = VectorDB() | |
| for i, entry in enumerate(data): | |
| text = entry["text"] | |
| vec = embedder.get_embedding(text) | |
| # Eğer vector_db.add_with_text fonksiyonu metadata kabul ediyorsa: | |
| vector_db.add_with_text(vec, text, metadata={ | |
| "category": entry["category"], | |
| "intent": entry["intent"], | |
| "flags": entry["flags"] | |
| }) | |
| if i % 100 == 0: | |
| vector_db.save() | |
| print(f"{i}/{len(data)} eklendi...") | |
| vector_db.save() | |
| print(f"{len(data)} metin vektör DB'ye eklendi ve kaydedildi.") | |
| if __name__ == "__main__": | |
| if not os.path.exists(CSV_PATH): | |
| print(f"CSV dosyası bulunamadı: {CSV_PATH}") | |
| else: | |
| build_vector_db_from_csv_with_meta(CSV_PATH) | |