# pip install -U transformers accelerate
# Load model directly
from transformers import AutoModelForSeq2SeqLM
model = AutoModelForSeq2SeqLM.from_pretrained("TrishanuDas/tayavision-multilingual", device_map="auto")Quick Links
TayaVision - Tiny Aya Vision (Instruct)
This model is an instruct sweep on liuhaotian/LLaVA-Instruct-150K with lora_a_lr_multiplier=1 and lora_b_lr_multiplier=0.1.
import torch
from PIL import Image
from transformers import AutoModelForCausalLM, AutoProcessor
repo = "TrishanuDas/tayavision-multilingual"
model = AutoModelForCausalLM.from_pretrained(repo, torch_dtype=torch.bfloat16, trust_remote_code=True)
model = model.to("cuda").eval()
processor = AutoProcessor.from_pretrained(repo, trust_remote_code=True)
image = Image.open("your_image.jpg").convert("RGB")
messages = [
{"role": "user", "content": [
{"type": "image"},
{"type": "text", "text": "Describe this image in detail."},
]},
]
inputs = processor.apply_chat_template(
messages,
images=image,
add_generation_prompt=True,
tokenize=True,
return_tensors="pt",
)
inputs = {key: value.to("cuda") for key, value in inputs.items()}
with torch.inference_mode():
output_ids = model.generate(
**inputs,
max_new_tokens=256,
do_sample=False,
use_cache=True,
)
response = processor.tokenizer.decode(
output_ids[0, inputs["input_ids"].shape[1]:],
skip_special_tokens=True,
)
print(response)
- Downloads last month
- 3
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="TrishanuDas/tayavision-multilingual") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)