{"architecture":{"backbone":"ViT-B/16 (224) + PubMedBERT (256 tokens)","family":"CLIP (contrastive image\u2013text)","input":"224\u00d7224 image / text \u2264 256 tokens","output":"512-dim joint embedding","params":"~196M","pretraining":"contrastive on PMC-15M"},"article":null,"cancer_slugs":["pan-cancer"],"category":"multimodal","confidence":"high","datasets":[{"name":"PubMed / PMC Open Access Subset","note":"","role":"pretraining","slug":"pubmed-pmc-oa"},{"name":"PatchCamelyon (PCam)","note":"","role":"evaluation","slug":"patchcamelyon"},{"name":"PubMed / PMC Open Access Subset","note":"","role":"training","slug":"pubmed-pmc-oa"}],"developer":"Microsoft Research","evaluation":[{"benchmark":"zero-shot classification on PCam, RSNA, and other public sets","dataset_slug":"patchcamelyon","external":true,"metric":"accuracy","source":"https://ai.nejm.org/doi/full/10.1056/AIoa2400640","value":"above general CLIP and PubMedCLIP (paper)"}],"hf":{"downloads":249031,"fetched_at":"2026-09-09T21:33:08Z","gated":false,"last_modified":"2025-01-14","library":"open_clip","license":"mit","likes":424,"pipeline_tag":"zero-shot-image-classification"},"kind":"foundation","license":"MIT","limitations":"- Trained on published figures, not clinical images: resolution, cropping and annotations differ from real slides and scans.\n- Captions carry publication bias.","links":{"demo":null,"docs":null,"doi":null,"github":"https://github.com/microsoft/BiomedCLIP_data_pipeline","huggingface":"https://huggingface.co/microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224","paper":"https://ai.nejm.org/doi/full/10.1056/AIoa2400640","pmid":null},"modalities":["multimodal","histopathology","radiology-xray","radiology-ct","literature"],"name":"BiomedCLIP","notable_uses":"","openness":"open-weights","regulatory":{"intended_use_en":"Research.","intended_use_pl":"Badania.","source_url":"https://huggingface.co/microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224","status":"research-only"},"regulatory_status":"research-only","release_date":"2023-03-02","run_snippet":"import open_clip\nmodel, preprocess = open_clip.create_model_from_pretrained('hf-hub:microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224')\ntokenizer = open_clip.get_tokenizer('hf-hub:microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224')\n# image_features = model.encode_image(preprocess(img).unsqueeze(0))\n# text_features = model.encode_text(tokenizer(['adenocarcinoma histopathology']))","settings":["basic-research","education"],"slug":"biomedclip","sources":[{"label":"Zhang S et al. BiomedCLIP: a multimodal biomedical foundation model pretrained from fifteen million scientific image-text pairs. NEJM AI 2025","url":"https://ai.nejm.org/doi/full/10.1056/AIoa2400640"},{"label":"Hugging Face \u2014 microsoft/BiomedCLIP","url":"https://huggingface.co/microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224"}],"summary":"CLIP-style vision-language model pretrained on PMC-15M \u2014 15 million figure\u2013caption pairs from biomedical papers \u2014 with a PubMedBERT text tower and a ViT-B image tower; supports zero-shot classification and retrieval across pathology, radiology and more.","tasks":["image-text-retrieval","classification","feature-extraction","question-answering"],"training":{"size":"15M image\u2013caption pairs","summary_en":"PMC-15M: 15 million figure\u2013caption pairs from 4.4 million PubMed Central open-access articles, spanning 30+ image types.","summary_pl":"PMC-15M: 15 mln par rycina\u2013opis z 4,4 mln otwartych artyku\u0142\u00f3w PubMed Central, obejmuj\u0105cych ponad 30 typ\u00f3w obraz\u00f3w."},"updated_at":"2026-09-09T21:33:09.703715","url":"/ai-oncology/models/biomedclip","usage":{"library":"open_clip","snippet":"import open_clip\nmodel, preprocess = open_clip.create_model_from_pretrained('hf-hub:microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224')\ntokenizer = open_clip.get_tokenizer('hf-hub:microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224')\n# image_features = model.encode_image(preprocess(img).unsqueeze(0))\n# text_features = model.encode_text(tokenizer(['adenocarcinoma histopathology']))"},"verified_at":"2026-09-05T22:26:00.027245","verified_by":"editorial","version":null,"what_it_does":"Embeds biomedical images and text into one space; strong general baseline when a domain-specific foundation model (e.g. CONCH for pathology) is unavailable or too restrictive to license."}