ai platform using llama-4-maverick 17b for image understanding
We have been dealing with alot of text completion and it is time that we try to use model with image. This is a sample code that we can use to extract contents from an image (invoice) and return line item in json format - so we can proceed it or fill up a form.
I assume you have enable llama-4-maverick serveless instance.
"""
Extract line items from an invoice image using Llama 4 Maverick on Vertex AI,
through the OpenAI-compatible Chat Completions endpoint.
Setup:
pip install openai pydantic google-auth requests
gcloud auth application-default login # or use a service account
Usage:
python extract_invoice_vertex.py invoice.jpg
"""
import base64
import json
import mimetypes
import re
import sys
from typing import Optional
import google.auth
import google.auth.transport.requests
from openai import OpenAI
from pydantic import BaseModel
PROJECT_ID = "YOUR-PROJECT-ID"
LOCATION = "us-east5"
MODEL = "meta/llama-4-maverick-17b-128e-instruct-maas"
# 1. Schema for the output --------------------------------------------------
class LineItem(BaseModel):
description: str
quantity: Optional[float] = None
unit_price: Optional[float] = None
total: Optional[float] = None
class Invoice(BaseModel):
vendor: Optional[str] = None
invoice_number: Optional[str] = None
invoice_date: Optional[str] = None
currency: Optional[str] = None
items: list[LineItem]
subtotal: Optional[float] = None
tax: Optional[float] = None
total: Optional[float] = None
# 2. Client (the access token expires after ~1 hour, so build it per run) ---
def get_client() -> OpenAI:
creds, _ = google.auth.default(
scopes=["https://www.googleapis.com/auth/cloud-platform"]
)
creds.refresh(google.auth.transport.requests.Request())
return OpenAI(
base_url=(
f"https://{LOCATION}-aiplatform.googleapis.com/v1/projects/"
f"{PROJECT_ID}/locations/{LOCATION}/endpoints/openapi"
),
api_key=creds.token,
)
# 3. Helpers ----------------------------------------------------------------
def image_to_data_url(path: str) -> str:
mime, _ = mimetypes.guess_type(path)
mime = mime or "image/jpeg"
with open(path, "rb") as f:
b64 = base64.b64encode(f.read()).decode("utf-8")
return f"data:{mime};base64,{b64}"
def parse_json(text: str) -> dict:
"""Pull a JSON object out of the reply, even if it's wrapped in ``` fences."""
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip())
start, end = text.find("{"), text.rfind("}")
return json.loads(text[start : end + 1])
# 4. Extraction -------------------------------------------------------------
PROMPT = f"""Extract the data from this invoice image.
Return ONLY a JSON object (no explanation, no markdown) matching this schema:
{json.dumps(Invoice.model_json_schema(), indent=2)}
Rules:
- Include every line item exactly as shown on the invoice.
- Numbers must be plain JSON numbers (no currency symbols or thousands separators).
- Use null for any field that is not present. Do not guess or invent values.
"""
def extract_invoice(path: str) -> Invoice:
client = get_client()
resp = client.chat.completions.create(
model=MODEL,
temperature=0,
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": PROMPT},
{"type": "image_url", "image_url": {"url": image_to_data_url(path)}},
],
}
],
# Optional: try uncommenting if your endpoint supports JSON mode
# response_format={"type": "json_object"},
)
raw = resp.choices[0].message.content
return Invoice.model_validate(parse_json(raw)) # validates types/fields
if __name__ == "__main__":
if len(sys.argv) != 2:
sys.exit("Usage: python extract_invoice_vertex.py <invoice_image>")
invoice = extract_invoice(sys.argv[1])
print(json.dumps(invoice.model_dump(), indent=2))
# Just the items:
# for item in invoice.items:
# print(item.description, item.quantity, item.total)
If all run well, then we would have the following outputs:-
Comments