Vision-Language Models

from transformers import CLIPProcessor, CLIPModel
import torch
import torch.nn.functional as F
from PIL import Image

model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
model.eval()

# Zero-shot classification
def zero_shot_classify(image: Image.Image, class_names: list[str]):
    texts = [f"a photo of a {c}" for c in class_names]
    inputs = processor(text=texts, images=image, return_tensors="pt", padding=True)
    with torch.no_grad():
        outputs = model(**inputs)
    img_emb  = F.normalize(outputs.image_embeds, dim=-1)
    text_emb = F.normalize(outputs.text_embeds,  dim=-1)
    logits = (img_emb @ text_emb.T) * model.logit_scale.exp()
    probs  = logits.softmax(dim=-1)[0]
    for name, p in sorted(zip(class_names, probs.tolist()), key=lambda x: -x[1])[:3]:
        print(f"{name:<20} {p:.3f}")

img = Image.open("test.jpg").convert("RGB")
zero_shot_classify(img, ["cat", "dog", "bird", "car", "bicycle", "airplane"])

# Image retrieval from text query
def build_image_index(image_paths: list[str], batch_size=32):
    all_embeds = []
    for i in range(0, len(image_paths), batch_size):
        batch = [Image.open(p).convert("RGB") for p in image_paths[i:i+batch_size]]
        inputs = processor(images=batch, return_tensors="pt", padding=True)
        with torch.no_grad():
            embeds = F.normalize(model.get_image_features(**inputs), dim=-1)
        all_embeds.append(embeds)
    return torch.cat(all_embeds)

def text_search(query: str, image_index: torch.Tensor, image_paths: list[str], top_k=5):
    inputs = processor(text=[query], return_tensors="pt", padding=True)
    with torch.no_grad():
        text_emb = F.normalize(model.get_text_features(**inputs), dim=-1)
    sims = (text_emb @ image_index.T)[0]
    top_idx = sims.topk(top_k).indices.tolist()
    return [(image_paths[i], sims[i].item()) for i in top_idx]

from transformers import Blip2Processor, Blip2ForConditionalGeneration
import torch
from PIL import Image

# BLIP-2 VQA
processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-2.7b")
model = Blip2ForConditionalGeneration.from_pretrained(
    "Salesforce/blip2-opt-2.7b", torch_dtype=torch.float16, device_map="auto",
)
model.eval()

image = Image.open("photo.jpg").convert("RGB")

def vqa_blip2(image, question):
    prompt = f"Question: {question} Answer:"
    inputs = processor(images=image, text=prompt, return_tensors="pt").to("cuda", torch.float16)
    out = model.generate(**inputs, max_new_tokens=30, num_beams=5)
    return processor.decode(out[0], skip_special_tokens=True).replace(prompt, "").strip()

for q in ["What is the main object?", "What color is it?", "How many are there?"]:
    print(f"Q: {q}  A: {vqa_blip2(image, q)}")

# LLaVA multi-turn conversation
from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration

lva_processor = LlavaNextProcessor.from_pretrained("llava-hf/llava-v1.6-mistral-7b-hf")
lva_model = LlavaNextForConditionalGeneration.from_pretrained(
    "llava-hf/llava-v1.6-mistral-7b-hf", torch_dtype=torch.float16, device_map="auto",
)

conversation = [{
    "role": "user",
    "content": [{"type": "image"}, {"type": "text", "text": "Describe this image in detail."}],
}]
prompt = lva_processor.apply_chat_template(conversation, add_generation_prompt=True)
inputs = lva_processor(images=image, text=prompt, return_tensors="pt").to("cuda")
output = lva_model.generate(**inputs, max_new_tokens=200, do_sample=False)
print(lva_processor.decode(output[0], skip_special_tokens=True))