"""End-to-end CLI behavior with Ollama faked and Claude forbidden.""" from __future__ import annotations import io import json import urllib.error from pathlib import Path import pypdfium2 as pdfium import pytest from click.testing import CliRunner from PIL import Image from pdf_tools import extract_scan, ollama @pytest.fixture def env(tmp_path, monkeypatch): root = tmp_path / "ACME" cfg = tmp_path / "scan-notes.toml" cfg.write_text(f""" prompt_extra = "Mark drawings." stats_log = "{tmp_path / "stats.jsonl"}" [[ollama.hosts]] name = "remote" url = "http://gpu-box:11434" model = "gemma4:31b" [[ollama.hosts]] name = "local" url = "http://localhost:11434" model = "gemma4:e4b" [[confidential]] root = "{root}" names = ["Board Person"] prompt_extra = "Board-only instruction." [confidential.companion] type = "meeting-notes" fields = ["body", "committee"] sections = ["Attendees", "Discussion", "Decisions", "Action items"] transcript_section = "Discussion" [[confidential.companion.paths]] match = "{{body}}/committees/{{committee}}/**" [[confidential]] root = "{tmp_path / "lab"}" names = ["Lab Person"] """) monkeypatch.setenv("PDF_TOOLS_SCAN_CONFIG", str(cfg)) monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) def no_claude(*a, **k): raise AssertionError("Claude must not be called") monkeypatch.setattr(extract_scan, "_transcribe_image", no_claude) folder = root / "ACMEP-BOM" / "committees" / "finance-audit" / "2026-10-29-fa" folder.mkdir(parents=True) pdf = folder / "2026-10-29-acmep-fa-notes.pdf" doc = pdfium.PdfDocument.new() doc.new_page(200, 260) doc.save(pdf) return {"root": root, "pdf": pdf, "tmp": tmp_path} def fake_ollama(monkeypatch, up: dict[str, list[str]]): def _fake(method, url, payload, timeout): base, _, path = url.partition("/api/") if path == "tags": if base not in up: raise OSError("unreachable") return {"models": [{"name": m} for m in up[base]]} if path == "ps": return {"models": [{"name": m, "size": 100, "size_vram": 100, "context_length": 4096} for m in up.get(base, [])]} _fake.prompts.append(payload["messages"][0]["content"]) return {"message": {"content": f"transcript via {payload['model']}"}, "prompt_eval_count": 1000, "eval_count": 200, "done_reason": "stop", "total_duration": 2_000_000_000} _fake.prompts = [] monkeypatch.setattr(ollama, "_http_json", _fake) return _fake def run(*args): return CliRunner().invoke(extract_scan.main, [str(a) for a in args]) def test_confidential_scan_defaults_to_ollama_and_writes_companion(env, monkeypatch): fake_ollama(monkeypatch, {"http://localhost:11434": ["gemma4:e4b"]}) r = run(env["pdf"]) assert r.exit_code == 0, r.output md = env["pdf"].with_suffix(".md").read_text() assert "body: ACMEP-BOM\ncommittee: finance-audit\ntype: meeting-notes" in md assert "## Discussion\n\n### Draft transcript (unverified)" in md assert "transcribed_by: ollama/localhost/gemma4:e4b" in md assert "transcript via gemma4:e4b" in md assert "falling back to local" in r.stderr def test_no_host_exits_without_cloud(env, monkeypatch): fake_ollama(monkeypatch, {}) monkeypatch.setenv("ANTHROPIC_API_KEY", "set-but-must-not-be-used") r = run(env["pdf"]) assert r.exit_code == extract_scan.EXIT_NO_HOST assert "Not falling back to a cloud service" in r.stderr assert not env["pdf"].with_suffix(".md").exists() def test_claude_refused_for_confidential_scan(env, monkeypatch): monkeypatch.setenv("ANTHROPIC_API_KEY", "x") r = run("--backend", "claude", env["pdf"]) assert r.exit_code == 2 and "--allow-cloud" in r.stderr def test_claude_refused_even_with_no_companion(env, monkeypatch): monkeypatch.setenv("ANTHROPIC_API_KEY", "x") r = run("--backend", "claude", "--no-companion", "--stdout", env["pdf"]) assert r.exit_code == 2 and "--allow-cloud" in r.stderr def test_forced_remote_fails_clearly(env, monkeypatch): fake_ollama(monkeypatch, {"http://localhost:11434": ["gemma4:e4b"]}) r = run("--host", "remote", env["pdf"]) assert r.exit_code == extract_scan.EXIT_NO_HOST assert "remote" in r.stderr and "falling back to local" not in r.stderr def test_model_flag_overrides_config(env, monkeypatch): fake_ollama(monkeypatch, {"http://gpu-box:11434": ["qwen3-vl:8b"]}) r = run("--model", "qwen3-vl:8b", "--host", "remote", env["pdf"]) assert r.exit_code == 0, r.output assert "ollama/gpu-box/qwen3-vl:8b" in env["pdf"].with_suffix(".md").read_text() def test_non_companion_ollama_output_uses_page_markers(env, monkeypatch): fake_ollama(monkeypatch, {"http://gpu-box:11434": ["gemma4:31b"]}) img = env["tmp"] / "page.png" Image.new("RGB", (50, 50), "white").save(img) r = run("--backend", "ollama", "--stdout", img) assert r.exit_code == 0, r.output assert "" in r.stdout assert "transcribed_by: ollama/gpu-box/gemma4:31b" in r.stdout def test_companion_mode_rejects_stdout(env): r = run("--stdout", env["pdf"]) assert r.exit_code == 2 and "--no-companion" in r.stderr def test_second_confidential_folder_uses_its_layout(env, monkeypatch): fake_ollama(monkeypatch, {"http://gpu-box:11434": ["gemma4:31b"]}) lab = env["tmp"] / "lab" / "reviews" lab.mkdir(parents=True) pdf = lab / "2026-11-02-review-notes.pdf" doc = pdfium.PdfDocument.new() doc.new_page(200, 260) doc.save(pdf) monkeypatch.setenv("ANTHROPIC_API_KEY", "x") assert run("--backend", "claude", pdf).exit_code == 2 r = run(pdf) assert r.exit_code == 0, r.output md = pdf.with_suffix(".md").read_text() assert "type: notes" in md and "## Notes" in md and "body:" not in md def test_mixing_confidential_and_other_files_refused(env, monkeypatch): img = env["tmp"] / "page.png" Image.new("RGB", (50, 50), "white").save(img) r = run("--backend", "ollama", "--no-companion", "--stdout", env["pdf"], img) assert r.exit_code == 2 and "do not mix" in r.stderr def test_names_come_only_from_the_scans_folder(env, monkeypatch): fake = fake_ollama(monkeypatch, {"http://localhost:11434": ["gemma4:e4b"]}) assert run(env["pdf"]).exit_code == 0 assert "- Board Person" in fake.prompts[0] assert "Lab Person" not in fake.prompts[0] assert "Mark drawings." in fake.prompts[0] assert "Board-only instruction." in fake.prompts[0] def test_only_top_level_settings_outside_confidential_folders(env, monkeypatch): fake = fake_ollama(monkeypatch, {"http://localhost:11434": ["gemma4:e4b"]}) img = env["tmp"] / "page.png" Image.new("RGB", (50, 50), "white").save(img) assert run("--backend", "ollama", "--stdout", img).exit_code == 0 assert fake.prompts[0] == ollama.build_prompt([], ["Mark drawings."]) assert "Board" not in fake.prompts[0] def test_no_confidential_folders_without_config(tmp_path, monkeypatch): monkeypatch.setenv("PDF_TOOLS_SCAN_CONFIG", str(tmp_path / "empty.toml")) (tmp_path / "empty.toml").write_text("") from pdf_tools.scan_config import load_config assert load_config().confidential == [] def test_interrupted_run_resumes_and_cleans_up(env, monkeypatch): nb = env["tmp"] / "notebooks" nb.mkdir() pdf = nb / "2024-04 notebook.pdf" doc = pdfium.PdfDocument.new() for _ in range(3): doc.new_page(200, 260) doc.save(pdf) calls = [] def _fake(method, url, payload, timeout): if url.endswith("/api/tags"): return {"models": [{"name": "gemma4:e4b"}]} if url.endswith("/api/ps"): return {"models": []} calls.append(1) if len(calls) == 2 and not getattr(_fake, "healthy", False): raise OSError("connection reset") return {"message": {"content": f"page text {len(calls)}"}} monkeypatch.setattr(ollama, "_http_json", _fake) r = run("--backend", "ollama", "--host", "local", pdf) assert r.exit_code == extract_scan.EXIT_TRANSCRIBE_FAILED assert "pages 1-1 are saved; rerun to resume" in r.stderr ckpt = nb / ".2024-04 notebook.scan-partial.json" assert ckpt.exists() and not pdf.with_suffix(".md").exists() _fake.healthy = True r = run("--backend", "ollama", "--host", "local", pdf) assert r.exit_code == 0, r.output assert "resuming after page 1 of 3" in r.stderr md = pdf.with_suffix(".md").read_text() assert "page text 1" in md and "" in md assert len(calls) == 4 # page 1 was not sent again assert not ckpt.exists() assert {p.name for p in nb.iterdir()} == {pdf.name, pdf.with_suffix(".md").name} def test_checkpoint_from_other_model_is_ignored(env, monkeypatch): fake_ollama(monkeypatch, {"http://localhost:11434": ["gemma4:e4b"]}) img = env["tmp"] / "page.png" Image.new("RGB", (50, 50), "white").save(img) ckpt = env["tmp"] / ".page.scan-partial.json" ckpt.write_text('{"key": {"transcribed_by": "other"}, "pages": [{"label": "x", "text": "stale"}]}') r = run("--backend", "ollama", "--stdout", img) assert r.exit_code == 0 and "stale" not in r.stdout assert "ignoring saved progress" in r.stderr and not ckpt.exists() def test_stats_logged_per_page_without_text(env, monkeypatch): fake_ollama(monkeypatch, {"http://localhost:11434": ["gemma4:e4b"]}) r = run(env["pdf"]) assert r.exit_code == 0, r.output assert "1200 tokens" in r.stderr lines = (env["tmp"] / "stats.jsonl").read_text().splitlines() assert len(lines) == 1 rec = json.loads(lines[0]) assert rec["model"] == "gemma4:e4b" and rec["host"] == "localhost" assert (rec["prompt_tokens"], rec["output_tokens"], rec["context_tokens"]) == (1000, 200, 1200) assert rec["gpu_fraction"] == 1.0 and rec["loaded_num_ctx"] == 4096 assert rec["image_px"][0] > 0 and rec["page"] == 1 assert "transcript via" not in lines[0] def test_warns_when_output_hits_length_limit(env, monkeypatch): def _fake(method, url, payload, timeout): if url.endswith("/api/tags"): return {"models": [{"name": "gemma4:e4b"}]} if url.endswith("/api/ps"): return {"models": [{"name": "gemma4:e4b", "size": 1, "size_vram": 1, "context_length": 4096}]} return {"message": {"content": "partial"}, "prompt_eval_count": 3900, "eval_count": 196, "done_reason": "length"} monkeypatch.setattr(ollama, "_http_json", _fake) r = run(env["pdf"]) assert r.exit_code == 0 assert "marked as cut off" in r.stderr assert "cut off at the output limit" in env["pdf"].with_suffix(".md").read_text() def test_looping_page_becomes_placeholder_and_run_finishes(env, monkeypatch): def _fake(method, url, payload, timeout): if url.endswith("/api/tags"): return {"models": [{"name": "gemma4:e4b"}]} if url.endswith("/api/ps"): return {"models": []} raise urllib.error.HTTPError( url, 500, "err", {}, io.BytesIO(b'{"error":"prediction aborted, token repeat limit reached"}')) monkeypatch.setattr(ollama, "_http_json", _fake) r = run(env["pdf"]) assert r.exit_code == 0, r.output assert "writing a placeholder" in r.stderr assert ollama.REPEAT_PLACEHOLDER in env["pdf"].with_suffix(".md").read_text() rec = json.loads((env["tmp"] / "stats.jsonl").read_text().splitlines()[0]) assert rec["done_reason"] == "repeat_abort" and rec["retries"] == 2 def test_ollama_is_the_default_backend_outside_confidential_folders(env, monkeypatch): fake_ollama(monkeypatch, {"http://gpu-box:11434": ["gemma4:31b"]}) monkeypatch.setenv("ANTHROPIC_API_KEY", "set-but-must-not-be-used") img = env["tmp"] / "page.png" Image.new("RGB", (50, 50), "white").save(img) r = run("--stdout", img) assert r.exit_code == 0, r.output assert "transcribed_by: ollama/gpu-box/gemma4:31b" in r.stdout