AI & LLM Integration: Local Ollama, OpenAI, and Anthropic Claude

Overview

dicomtk connects DICOM metadata and pixel data to both local LLMs (via Ollama) and OpenAI-compatible cloud or subscription endpoints (OpenAI, Anthropic Claude, OpenRouter, LM Studio, vLLM, Azure OpenAI).

Configuring Providers with dicom_llm()

library(dicomtk)

# 1. Local Ollama (100% private, runs on your machine)
ollama_llm <- dicom_llm(
  provider = "ollama",
  model = "llava:13b",
  base_url = "http://localhost:11434"
)
ollama_llm
#> ℹ DICOM LLM Provider: ollama (model: llava:13b, endpoint: http://localhost:11434, local: TRUE)

# 2. OpenAI or any OpenAI-compatible API endpoint
openai_llm <- dicom_llm(
  provider = "openai",
  model = "gpt-4o",
  base_url = "https://api.openai.com/v1"
)

# 3. Anthropic Claude
claude_llm <- dicom_llm(
  provider = "anthropic",
  model = "claude-sonnet-4-20250514"
)

# 4. Deterministic offline provider (for reproducible pipelines without network access)
offline_llm <- dicom_llm("offline")

Summarizing Studies, Describing Images & Generating Structured Reports

dcm_file <- system.file("extdata", "example.dcm", package = "dicomtk")

# Natural-language study metadata summary
dicom_summarize(dcm_file, llm = offline_llm)
#> [1] "DICOM Study Summary: CT imaging study (General Study) acquired on GE MEDICAL SYSTEMS system. Series: SmartScore - Gated 0.5 sec (Body Part: Not specified). Image Matrix: 512 x 512 pixels (16-bit MONOCHROME2)."

# Vision + quantitative pixel analysis
dicom_describe_image(dcm_file, llm = offline_llm)
#> [1] "DICOM Image Analysis (CT, 512x512, Photometric: MONOCHROME2): Rescaled pixel intensities range from -3024 to 1483 (mean: -704.75, SD: 1230.16, median: -78; 5th-95th percentile: [-3024, 102]). Window Center: 40, Window Width: 400."

# Generate a structured radiology report and export as a DICOM Structured Report (.dcm)
sr_out <- tempfile(fileext = ".dcm")
report_tbl <- dicom_report(dcm_file, llm = offline_llm, save_sr_path = sr_out)
report_tbl
#> # A tibble: 4 × 2
#>   section     content                                                           
#>   <chr>       <chr>                                                             
#> 1 EXAMINATION CT -  - SmartScore - Gated 0.5 sec                                
#> 2 TECHNIQUE   DICOM Study Summary: CT imaging study (General Study) acquired on…
#> 3 FINDINGS    DICOM Image Analysis (CT, 512x512, Photometric: MONOCHROME2): Res…
#> 4 IMPRESSION  Automated quantitative and metadata review completed. Correlate c…

# Read back the generated DICOM Structured Report
read_dicom_sr(sr_out)
#> # A tibble: 14 × 6
#>    depth sequence_path tag         keyword              vr    value             
#>    <int> <chr>         <chr>       <chr>                <chr> <chr>             
#>  1     0 ""            (0008,0005) SpecificCharacterSet CS    "ISO_IR 192"      
#>  2     0 ""            (0008,0016) SOPClassUID          UI    "1.2.840.10008.5.…
#>  3     0 ""            (0008,0018) SOPInstanceUID       UI    "2.25.18455343503…
#>  4     0 ""            (0008,0060) Modality             CS    "SR"              
#>  5     0 ""            (0008,103E) SeriesDescription    LO    "AI Structured Re…
#>  6     0 ""            (0010,0010) PatientName          PN    "ANONYMOUS"       
#>  7     0 ""            (0010,0020) PatientID            LO    "ANON001"         
#>  8     0 ""            (0020,000D) StudyInstanceUID     UI    "1.3.6.1.4.1.5962…
#>  9     0 ""            (0020,000E) SeriesInstanceUID    UI    "2.25.16628048664…
#> 10     0 ""            (0020,0013) InstanceNumber       IS    "1"               
#> 11     0 ""            (0040,A040) ValueType            CS    "TEXT"            
#> 12     0 ""            (0040,A160) TextValue            UT    "EXAMINATION:\nCT…
#> 13     0 ""            (0040,A491) CompletionFlag       CS    "COMPLETE"        
#> 14     0 ""            (0040,A493) VerificationFlag     CS    "UNVERIFIED"

Natural-Language DICOM Q&A and Tag Lookup

dicom_ask(dcm_file, "What are the image dimensions and photometric interpretation?", llm = offline_llm)
#> [1] "Photometric Interpretation ((0028,0004)): MONOCHROME2; Image Type ((0008,0008)): ORIGINAL\\PRIMARY\\AXIAL; Image Position (Patient) ((0020,0032)): -72.199997\\-143.000000\\-101.237503; Image Orientation (Patient) ((0020,0037)): 1.000000\\0.000000\\0.000000\\0.000000\\1.000000\\0.000000"

# Look up standard DICOM tags with natural language
dicom_tag_lookup("What tag stores window center?", llm = offline_llm)
#> # A tibble: 25 × 5
#>    tag         name                                    keyword       vr    vm   
#>    <chr>       <chr>                                   <chr>         <chr> <chr>
#>  1 (0028,1050) Window Center                           WindowCenter  DS    1-n  
#>  2 (0028,1055) Window Center & Width Explanation       WindowCenter… LO    1-n  
#>  3 (0018,0032) Energy Window Centerline                RETIRED_Ener… DS    1    
#>  4 (0012,0060) Clinical Trial Coordinating Center Name ClinicalTria… LO    1    
#>  5 (0014,401B) Measured Center Frequency               MeasuredCent… DS    1    
#>  6 (0014,4058) Probe Center Location X                 ProbeCenterL… DS    1    
#>  7 (0014,4059) Probe Center Location Z                 ProbeCenterL… DS    1    
#>  8 (0014,604C) Moving Window Size                      MovingWindow… DS    1    
#>  9 (0014,604D) Moving Window Type                      MovingWindow… CS    1    
#> 10 (0014,604E) Moving Window Weights                   MovingWindow… DS    1-n  
#> # ℹ 15 more rows