app.py

import gradio as gr
from datasets import load_dataset
from transformers import CLIPTokenizerFast, CLIPProcessor, CLIPModel
import torch
from tqdm.auto import tqdm
import numpy as np
import time

device = 'cpu' # 'cuda' if torch.cuda.is_available() else "cpu"
model_id = 'openai/clip-vit-base-patch32'
model = CLIPModel.from_pretrained(model_id).to(device)
tokenizer = CLIPTokenizerFast.from_pretrained(model_id)
processor = CLIPProcessor.from_pretrained(model_id)



def load_data():
    global imagenette
    imagenette = load_dataset(
        'frgfm/imagenette',
        'full_size',
        split = 'train',
        ignore_verifications = False # set to True if seeing splits Error
    )
    return imagenette

def embedding_input(text_input):
    token_input = tokenizer(text_input, return_tensors = "pt")
    text_emb = model.get_text_features(**token_input.to(device))
    return text_emb

def embedding_img():
    global images, image_arr
    load_data()
    sample_idx= np.random.randint(0, len(imagenette)+1, 100).tolist()
    images = [imagenette[i]['image'] for i in sample_idx]
    batch_sie = 5
    image_arr = None
    for i in tqdm(range(0, len(images), batch_sie)):
        time.sleep(1)
        batch = images[i:i+batch_sie]

        batch = processor(
            text = None,
            images = batch,
            return_tensors= 'pt',
            padding = True
        )['pixel_values'].to(device)
        batch_emb = model.get_image_features(pixel_values = batch)
        batch_emb = batch_emb.squeeze(0)
        batch_emb = batch_emb.cpu().detach().numpy()
        
        if image_arr is None:
            image_arr = batch_emb
            
        else:
            image_arr = np.concatenate((image_arr, batch_emb), axis = 0)
    return image_arr

def norm_val(text_input):
    text_emb = embedding_input(text_input)
    image_emb = (image_arr.T / np.linalg.norm(image_arr, axis = 1)).T
    text_emb = text_emb.cpu().detach().numpy()
    scores = np.dot(text_emb, image_emb.T)
    top_k = 1
    idx = np.argsort(-scores[0])[:top_k]
    return images[idx[0]]
        




if __name__ == "__main__":
    embedding_img()
    load_data()
    iface = gr.Interface(fn=norm_val, inputs="text", outputs="image")
    iface.launch(inline = False )