import gradio as gr import torch import sys import os import sentencepiece as spm # Add local path to import custom model sys.path.append(os.path.join(os.path.dirname(__file__), "model")) from modeling_ngwanda import NgwandaModel # Load the SentencePiece tokenizer directly sp = spm.SentencePieceProcessor(model_file="vocab_12k/corpus_tokenized_12000.model") bos_token = "<|startoftext|>" bos_id = sp.piece_to_id(bos_token) # Load your Ngwanda base model model = NgwandaModel.from_pretrained("takuM23/ShonaTransformer-Basemodel") def predict(text, tokens, temp): print(text) # encode as IDs input_ids = sp.encode_as_ids(text) if not text.startswith(bos_token): input_ids = [bos_id] + input_ids inputs_tensor = torch.tensor([input_ids]) print({'input_ids': inputs_tensor}) with torch.no_grad(): outputs = model.generate( input_ids=inputs_tensor, max_new_tokens=int(tokens), temperature=float(temp), do_sample=True, ) out_ids = outputs[0].tolist() # filter out special tokens manually special_ids = {sp.bos_id(), sp.eos_id(), sp.pad_id(), sp.unk_id(), sp.piece_to_id("<|startoftext|>"), sp.piece_to_id("<|endofturn|>"), sp.piece_to_id("<|user|>"), sp.piece_to_id("<|agent|>")} out_ids = [i for i in out_ids if i not in special_ids] # decode the original input plus newly generated out_text = sp.decode(out_ids) print("Otput text ", out_text) return out_text # Gradio Interface (Text Completion style) gr.Interface( fn=predict, inputs=[ gr.Textbox(label="Shona Input"), gr.Slider(10, 200, value=160, step=1, label="Max New Tokens"), gr.Slider(0.1, 1.0, value=0.9, step=0.1, label="Temperature"), ], outputs="text", title="Shona LLM (BaseModel)", description="75.1M parameter transformer model trained on raw shona text" ).launch()