Decision models on images: classify receipts, screenshots and forms locally with Clef

ollama -v          # 0.35.1 or later
ollama pull clef   # or: ollama pull clef-flash

curl http://localhost:11434/v1/systemone \
  -H 'Content-Type: application/json' \
  -d '{
    "model": "clef",
    "state": "Receipt from a cafe: two coffees.",
    "questions": {
      "meal": {"type": "noul", "instructions": "Is this a food or drink purchase?"}
    }
  }'

# request.json: the image goes in "images", the text goes in "state"
{
  printf '{"model":"clef","state":"Expense receipt photographed by an employee.","images":["'
  base64 < receipt.jpg | tr -d '\n'
  printf '"],"questions":{"legible":{"type":"noul","instructions":"Is the receipt total fully legible?"},"category":{"type":"choice","instructions":"What kind of purchase is this?","criteria":{"meals":"Restaurants, cafes, food delivery","travel":"Transport, fuel, parking, accommodation","office":"Stationery, equipment, software","other":"Anything else"}}}}'
} > request.json

curl http://localhost:11434/v1/systemone \
  -H 'Content-Type: application/json' \
  --data-binary @request.json

# "$PWD\receipt.jpg": .NET resolves relative paths against the process folder, not your PowerShell location
$image = [Convert]::ToBase64String([IO.File]::ReadAllBytes("$PWD\receipt.jpg"))
$body = [ordered]@{
    model     = "clef"
    state     = "Expense receipt photographed by an employee."
    images    = @($image)
    questions = [ordered]@{
        legible  = [ordered]@{ type = "noul"; instructions = "Is the receipt total fully legible?" }
        category = [ordered]@{
            type         = "choice"
            instructions = "What kind of purchase is this?"
            criteria     = [ordered]@{
                meals  = "Restaurants, cafes, food delivery"
                travel = "Transport, fuel, parking, accommodation"
                office = "Stationery, equipment, software"
                other  = "Anything else"
            }
        }
    }
} | ConvertTo-Json -Depth 8
Invoke-RestMethod -Uri "http://localhost:11434/v1/systemone" -Method Post -ContentType "application/json" -Body $body | ConvertTo-Json -Depth 8

{
  "model": "clef",
  "answers": {
    "legible": {"type": "noul", "noul": 0.93},
    "category": {
      "type": "choice",
      "choice": "meals",
      "probabilities": {"meals": 0.88, "travel": 0.06, "office": 0.04, "other": 0.02},
      "confidence": 0.648
    }
  }
}

"""decision_images.py - helpers for sending images to Ollama's /v1/systemone (Clef, Clef-Flash).

Needs Python 3.9+ and Pillow:  pip install pillow pillow-heif
(pillow-heif is optional; it lets Pillow open iPhone HEIC photos.)
"""
import base64
import io
import json
import os
import urllib.error
import urllib.request

from PIL import Image, ImageOps

try:
    from pillow_heif import register_heif_opener

    register_heif_opener()  # teaches Pillow to open .heic / .heif files
except ImportError:
    pass  # without it, HEIC files raise an error in encode_image()

BASE_URL = os.environ.get("OLLAMA_HOST", "127.0.0.1:11434").replace("0.0.0.0", "127.0.0.1")
if "://" not in BASE_URL:
    BASE_URL = "http://" + BASE_URL

TEXT_LIMIT = 64 * 1024          # request body limit without images (Ollama API reference)
IMAGE_LIMIT = 32 * 1024 * 1024  # request body limit with images, JSON and base64 included


class DecisionError(Exception):
    def __init__(self, status, message):
        super().__init__(f"HTTP {status}: {message}")
        self.status = status


def encode_image(path, max_side=1600, fmt="JPEG", quality=85):
    """Open any image Pillow can read and return base64 text that /v1/systemone accepts.

    - applies the phone's EXIF rotation, so the model sees the photo upright
    - shrinks the longest side to max_side (never enlarges)
    - flattens transparency onto white and converts to RGB
    - writes PNG, JPEG or WEBP, the only formats Ollama accepts
    - copies no metadata: GPS position and camera details stay behind
    """
    with Image.open(path) as im:
        im = ImageOps.exif_transpose(im)
        im.thumbnail((max_side, max_side))
        if im.mode in ("RGBA", "LA") or (im.mode == "P" and "transparency" in im.info):
            im = im.convert("RGBA")
            flat = Image.new("RGB", im.size, "white")
            flat.paste(im, mask=im.getchannel("A"))
            im = flat
        else:
            im = im.convert("RGB")
        buf = io.BytesIO()
        options = {"quality": quality} if fmt in ("JPEG", "WEBP") else {}
        im.save(buf, fmt, **options)
    return base64.b64encode(buf.getvalue()).decode("ascii")


def decide(model, state, questions, images=None, keep_alive="10m", timeout=300):
    """POST one decision request. Raises DecisionError on any HTTP error."""
    body = {"model": model, "state": state, "questions": questions, "keep_alive": keep_alive}
    if images:
        body["images"] = images
    raw = json.dumps(body).encode("utf-8")
    limit = IMAGE_LIMIT if images else TEXT_LIMIT
    if len(raw) > limit:
        raise DecisionError(413, f"body is {len(raw):,} bytes, over the {limit:,}-byte limit; shrink it before sending")
    request = urllib.request.Request(
        BASE_URL + "/v1/systemone", data=raw, headers={"Content-Type": "application/json"}
    )
    try:
        with urllib.request.urlopen(request, timeout=timeout) as response:
            return json.loads(response.read())
    except urllib.error.HTTPError as err:
        detail = err.read().decode("utf-8", "replace").strip()
        raise DecisionError(err.code, detail) from None

# triage.py - sort expense-receipt photos with Clef
import sys
from decision_images import decide, encode_image

QUESTIONS = {
    "legible": {
        "type": "noul",
        "instructions": "Is the receipt total fully legible?",
    },
    "cut_off": {
        "type": "noul",
        "instructions": "Is any part of the receipt cut off or missing from the photo?",
    },
    "category": {
        "type": "choice",
        "instructions": "What kind of purchase is this?",
        "criteria": {
            "meals": "Restaurants, cafes, food delivery",
            "travel": "Transport, fuel, parking, accommodation",
            "office": "Stationery, equipment, software",
            "other": "Anything else",
        },
    },
}


def triage(path, model="clef"):
    result = decide(
        model,
        "Expense receipt photographed by an employee.",
        QUESTIONS,
        images=[encode_image(path)],
    )
    answers = result["answers"]
    legible = answers["legible"]["noul"]
    cut_off = answers["cut_off"]["noul"]
    category = answers["category"]["choice"]
    category_p = answers["category"]["probabilities"][category]

    # Starting thresholds, not recommendations. Measure your own (see the calibration guide).
    if legible < 0.80 or cut_off > 0.30:
        return "retake", {"legible": legible, "cut_off": cut_off}
    if category_p < 0.70:
        return "human_review", {"category": category, "p": category_p}
    return "auto_file", {"category": category, "p": category_p}


if __name__ == "__main__":
    for path in sys.argv[1:]:
        action, detail = triage(path)
        print(f"{path}: {action} {detail}")

# in eval_decisions.py: add `from decision_images import encode_image`, then replace the requests.post call with
r = requests.post(URL, json={
    "model": model,
    "state": ex["state"],
    "images": [encode_image(ex["image"])],
    "questions": {"q": QUESTION},
    "keep_alive": "10m",
}, timeout=300)