pdf-tools/tests/test_ollama_hosts.py
Eric Furst e464871b69 Initial public release of pdf-tools
Command-line tools for marked-up PDFs and scanned handwritten notes:
pdf-scrub, pdf-notes-md, scan-notes-md (Ollama by default, Claude on
request), scan-notes-index (per-page search metadata), and
scan-notes-stats.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011ZSvbNaLhgFuTm7wmkgbWs
2026-11-03 07:25:51 -05:00

247 lines
9.8 KiB
Python

"""Host selection, fallback, and response cleanup. All HTTP is faked."""
from __future__ import annotations
import io
import json
import urllib.error
import pytest
from pdf_tools import ollama
from pdf_tools.scan_config import OllamaHost
REMOTE = OllamaHost("remote", "http://gpu-box:11434", "gemma4:31b")
LOCAL = OllamaHost("local", "http://localhost:11434", "gemma4:e4b")
def fake_http(tags: dict[str, list[str] | None], chat=None):
"""tags maps base URL -> model names, or None for an unreachable host."""
calls = []
def _fake(method, url, payload, timeout):
calls.append((method, url, payload, timeout))
base, _, path = url.partition("/api/")
if path == "tags":
models = tags.get(base)
if models is None:
raise urllib.error.URLError("connection refused")
return {"models": [{"name": m} for m in models]}
if path == "chat":
return chat(payload) if chat else {"message": {"content": "ok"}}
raise AssertionError(url)
_fake.calls = calls
return _fake
def test_remote_used_when_available(monkeypatch, capsys):
monkeypatch.setattr(ollama, "_http_json", fake_http({
REMOTE.url: ["gemma4:31b"], LOCAL.url: ["gemma4:e4b"]}))
assert ollama.select_host([REMOTE, LOCAL], 2.5) is REMOTE
assert capsys.readouterr().err == ""
def test_falls_back_when_remote_unreachable(monkeypatch, capsys):
monkeypatch.setattr(ollama, "_http_json", fake_http({
REMOTE.url: None, LOCAL.url: ["gemma4:e4b"]}))
assert ollama.select_host([REMOTE, LOCAL], 2.5) is LOCAL
err = capsys.readouterr().err.strip().splitlines()
assert len(err) == 1
assert "unreachable" in err[0] and "falling back to local" in err[0]
def test_falls_back_when_remote_lacks_model(monkeypatch, capsys):
monkeypatch.setattr(ollama, "_http_json", fake_http({
REMOTE.url: ["qwen3-vl:8b"], LOCAL.url: ["gemma4:e4b"]}))
assert ollama.select_host([REMOTE, LOCAL], 2.5) is LOCAL
assert "does not have gemma4:31b" in capsys.readouterr().err
def test_probe_uses_short_timeout(monkeypatch):
fake = fake_http({REMOTE.url: ["gemma4:31b"]})
monkeypatch.setattr(ollama, "_http_json", fake)
ollama.select_host([REMOTE, LOCAL], 2.5)
assert fake.calls == [("GET", f"{REMOTE.url}/api/tags", None, 2.5)]
def test_no_host_raises(monkeypatch):
monkeypatch.setattr(ollama, "_http_json", fake_http({}))
with pytest.raises(ollama.NoHostAvailable):
ollama.select_host([REMOTE, LOCAL], 2.5)
def test_forced_host_skips_others(monkeypatch):
fake = fake_http({REMOTE.url: ["gemma4:31b"], LOCAL.url: ["gemma4:e4b"]})
monkeypatch.setattr(ollama, "_http_json", fake)
assert ollama.select_host([REMOTE, LOCAL], 2.5, forced="local") is LOCAL
assert [c[1] for c in fake.calls] == [f"{LOCAL.url}/api/tags"]
def test_forced_host_unavailable_is_error_not_fallback(monkeypatch):
monkeypatch.setattr(ollama, "_http_json", fake_http({
REMOTE.url: None, LOCAL.url: ["gemma4:e4b"]}))
with pytest.raises(ollama.NoHostAvailable, match="remote"):
ollama.select_host([REMOTE, LOCAL], 2.5, forced="remote")
def test_forced_host_unknown_name(monkeypatch):
monkeypatch.setattr(ollama, "_http_json", fake_http({}))
with pytest.raises(ollama.NoHostAvailable, match="no Ollama host named"):
ollama.select_host([REMOTE, LOCAL], 2.5, forced="laptop")
def test_untagged_model_matches_latest(monkeypatch):
host = OllamaHost("x", "http://x:11434", "llama3.2")
monkeypatch.setattr(ollama, "_http_json", fake_http({host.url: ["llama3.2:latest"]}))
assert ollama.select_host([host], 1) is host
def test_chat_request_shape(monkeypatch):
seen = {}
def chat(payload):
seen.update(payload)
return {"message": {"content": "line one\n line two"}}
monkeypatch.setattr(ollama, "_http_json", fake_http({}, chat))
out = ollama.transcribe_png(b"\x89PNG", REMOTE, "PROMPT", 300)
assert out.text == "line one\n line two"
assert seen["stream"] is False and seen["think"] is False
assert seen["options"]["temperature"] == 0
assert seen["model"] == "gemma4:31b"
assert len(seen["messages"]) == 1 and seen["messages"][0]["images"]
def test_retries_without_think_when_rejected(monkeypatch):
payloads = []
def _fake(method, url, payload, timeout):
payloads.append(dict(payload))
if "think" in payload:
raise urllib.error.HTTPError(
url, 400, "Bad Request", {},
io.BytesIO(json.dumps({"error": "model does not support thinking"}).encode()))
return {"message": {"content": "text"}}
monkeypatch.setattr(ollama, "_http_json", _fake)
assert ollama.transcribe_png(b"x", LOCAL, "P", 10).text == "text"
assert "think" in payloads[0] and "think" not in payloads[1]
@pytest.mark.parametrize("raw, clean", [
("<think>hmm, looks like a list</think>\nA\nB", "A\nB"),
("<thinking>x</thinking>A", "A"),
("reasoning without an opening tag</think>\nA", "A"),
("A\n - indented\nB\n", "A\n - indented\nB"),
])
def test_strip_thinking(raw, clean):
assert ollama.strip_thinking(raw) == clean
def test_names_added_only_when_configured():
assert ollama.build_prompt([]) == ollama.HANDWRITING_PROMPT
p = ollama.build_prompt(["Pat Example", "ACME Foundation"])
assert p.startswith(ollama.HANDWRITING_PROMPT)
assert "- Pat Example" in p and "otherwise write\n[illegible]" in p
def test_provenance():
assert ollama.provenance(REMOTE) == "ollama/gpu-box/gemma4:31b"
assert ollama.provenance(LOCAL) == "ollama/localhost/gemma4:e4b"
def test_extras_go_between_base_prompt_and_names():
p = ollama.build_prompt(["Pat"], ["Draw figures as [FIGURE: x].", " "])
assert p.startswith(ollama.HANDWRITING_PROMPT + "\nDraw figures as [FIGURE: x].\n\n")
assert p.index("[FIGURE") < p.index(ollama.NAMES_PROMPT) < p.index("- Pat")
assert ollama.build_prompt([], ["", "\n"]) == ollama.HANDWRITING_PROMPT
def test_page_stats_from_response():
st = ollama.page_stats({"prompt_eval_count": 1200, "eval_count": 300,
"done_reason": "stop", "total_duration": 9_500_000_000,
"load_duration": 0, "prompt_eval_duration": 1_000_000_000,
"eval_duration": 8_000_000_000})
assert st["context_tokens"] == 1500 and st["done_reason"] == "stop"
assert (st["total_s"], st["prompt_s"], st["output_s"]) == (9.5, 1.0, 8.0)
assert ollama.page_stats({})["context_tokens"] == 0
def test_loaded_model_reports_gpu_fit(monkeypatch):
def _fake(method, url, payload, timeout):
assert url.endswith("/api/ps")
return {"models": [{"name": "gemma4:31b", "size": 21_000_000_000,
"size_vram": 19_740_000_000, "context_length": 32768}]}
monkeypatch.setattr(ollama, "_http_json", _fake)
got = ollama.loaded_model(REMOTE, 2)
assert got == {"loaded_bytes": 21_000_000_000, "vram_bytes": 19_740_000_000,
"gpu_fraction": 0.94, "loaded_num_ctx": 32768}
def test_loaded_model_unreachable_is_empty(monkeypatch):
def _down(*a):
raise urllib.error.URLError("connection refused")
monkeypatch.setattr(ollama, "_http_json", _down)
assert ollama.loaded_model(REMOTE, 2) == {}
def _repeat_error(url):
return urllib.error.HTTPError(
url, 500, "Internal Server Error", {},
io.BytesIO(b'{"error":"prediction aborted, token repeat limit reached"}'))
def test_looping_page_retried_with_sampling(monkeypatch):
seen = []
def _fake(method, url, payload, timeout):
seen.append(dict(payload["options"]))
if len(seen) == 1:
raise _repeat_error(url)
return {"message": {"content": "ok"}, "eval_count": 5}
monkeypatch.setattr(ollama, "_http_json", _fake)
out = ollama.transcribe_png(b"x", REMOTE, "P", 10)
assert out.text == "ok" and out.stats["retries"] == 1
assert out.stats["retry_reasons"] == ["repeat_limit"]
assert seen[0] == {"temperature": 0}
assert seen[1] == {"temperature": 0.3, "repeat_penalty": 1.15}
assert out.stats["retry_options"] == ollama.REPEAT_RETRIES[0]
def test_first_try_keeps_temperature_zero(monkeypatch):
seen = []
monkeypatch.setattr(ollama, "_http_json", lambda m, u, p, t: seen.append(p["options"])
or {"message": {"content": "ok"}})
out = ollama.transcribe_png(b"x", REMOTE, "P", 10)
assert seen == [{"temperature": 0}] and out.stats["retries"] == 0
def test_repetition_error_after_all_retries(monkeypatch):
def _fake(method, url, payload, timeout):
raise _repeat_error(url)
monkeypatch.setattr(ollama, "_http_json", _fake)
with pytest.raises(ollama.RepetitionError):
ollama.transcribe_png(b"x", REMOTE, "P", 10)
def test_runaway_output_capped_and_retried(monkeypatch):
seen = []
def _fake(method, url, payload, timeout):
seen.append(dict(payload["options"]))
if len(seen) == 1:
return {"message": {"content": "x x x"}, "done_reason": "length", "eval_count": 2048}
return {"message": {"content": "fine"}, "done_reason": "stop", "eval_count": 300}
monkeypatch.setattr(ollama, "_http_json", _fake)
out = ollama.transcribe_png(b"x", REMOTE, "P", 10, max_output=2048)
assert seen[0] == {"temperature": 0, "num_predict": 2048}
assert out.text == "fine" and out.stats["retries"] == 1
assert out.stats["retry_reasons"] == ["output_cap"]
def test_runaway_on_every_attempt_is_marked_cut_off(monkeypatch):
monkeypatch.setattr(ollama, "_http_json", lambda m, u, p, t: {
"message": {"content": "long"}, "done_reason": "length", "eval_count": 2048})
out = ollama.transcribe_png(b"x", REMOTE, "P", 10, max_output=2048)
assert out.text == "long\n" + ollama.CUT_OFF_MARKER
assert out.stats["retries"] == 2 and out.stats["done_reason"] == "length"