local llm image understanding using GLM OCR

GLM OCR is pretty lightweight and fit for purpose when i tried it out on invoice images.  Here is the code for setting up and running it locally. 

"""

Extract line items from an invoice image using a local model served at

http://localhost:8888/v1, called through LiteLLM.

Setup:

    pip install litellm pydantic

    export LOCAL_API_KEY="your-local-server-key"

Usage:

    python extract_invoice_local.py invoice.jpg

"""

import base64

import json

import mimetypes

import os

import re

import sys

from typing import Optional

import litellm

from pydantic import BaseModel

# Same settings as your LiteLlm config ---------------------------------------

MODEL = "openai/ggml-org/GLM-OCR-GGUF"

API_BASE = "http://localhost:8888/v1"

API_KEY = "sk-unsloth-4d0a1b198bd177a2a72ee1954585342a"  # keep the key out of source code

TEMPERATURE = 0.0

# Ask the server to constrain output to the schema (llama.cpp-based servers

# support this). If your server rejects it, the script retries without it.

TRY_SCHEMA_CONSTRAINT = True

# 1. Schema for the output ---------------------------------------------------

class LineItem(BaseModel):

    description: str

    quantity: Optional[float] = None

    unit_price: Optional[float] = None

    total: Optional[float] = None

class Invoice(BaseModel):

    vendor: Optional[str] = None

    invoice_number: Optional[str] = None

    invoice_date: Optional[str] = None

    currency: Optional[str] = None

    items: list[LineItem]

    subtotal: Optional[float] = None

    tax: Optional[float] = None

    total: Optional[float] = None

# 2. Helpers -----------------------------------------------------------------

def image_to_data_url(path: str) -> str:

    mime, _ = mimetypes.guess_type(path)

    mime = mime or "image/jpeg"

    with open(path, "rb") as f:

        b64 = base64.b64encode(f.read()).decode("utf-8")

    return f"data:{mime};base64,{b64}"

def parse_json(text: str) -> dict:

    """Pull a JSON object out of the reply, tolerating <think> blocks and ``` fences."""

    text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL)  # Qwen reasoning

    text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip())

    start, end = text.find("{"), text.rfind("}")

    return json.loads(text[start : end + 1])

# 3. Extraction --------------------------------------------------------------

PROMPT = f"""Extract the data from this invoice image.

Return ONLY a JSON object (no explanation, no markdown) matching this schema:

{json.dumps(Invoice.model_json_schema(), indent=2)}

Rules:

- Include every line item exactly as shown on the invoice.

- Numbers must be plain JSON numbers (no currency symbols or thousands separators).

- Use null for any field that is not present. Do not guess or invent values.

"""

def call_model(messages: list, constrain: bool):

    kwargs = dict(

        model=MODEL,

        api_base=API_BASE,

        api_key=API_KEY,

        temperature=TEMPERATURE,

        max_tokens=4096,

        timeout=300,  # local models can be slow on large images

        messages=messages,

    )

    if constrain:

        kwargs["response_format"] = {

            "type": "json_schema",

            "json_schema": {

                "name": "invoice",

                "schema": Invoice.model_json_schema(),

                "strict": True,

            },

        }

    return litellm.completion(**kwargs)

def extract_invoice(path: str) -> Invoice:

    messages = [

        {

            "role": "user",

            "content": [

                {"type": "text", "text": PROMPT},

                {"type": "image_url", "image_url": {"url": image_to_data_url(path)}},

            ],

        }

    ]

    try:

        resp = call_model(messages, constrain=TRY_SCHEMA_CONSTRAINT)

    except Exception as e:

        if not TRY_SCHEMA_CONSTRAINT:

            raise

        print(f"[warn] schema-constrained call failed ({e}); retrying without it",

              file=sys.stderr)

        resp = call_model(messages, constrain=False)

    raw = resp.choices[0].message.content

    return Invoice.model_validate(parse_json(raw))  # validates types/fields

if __name__ == "__main__":

    if len(sys.argv) != 2:

        sys.exit("Usage: python extract_invoice_local.py <invoice_image>")

    invoice = extract_invoice(sys.argv[1])

    print(json.dumps(invoice.model_dump(), indent=2))

    # Just the items:

    # for item in invoice.items:

    #     print(item.description, item.quantity, item.total)

And the output are given as below. Noticed that the vendor is null. It is fall abit short compare to LLama maverick model.