"""scan-notes-index: transcript parsing, page files, and the CLI with Ollama faked.""" from __future__ import annotations import io import json import urllib.error import pypdfium2 as pdfium import pytest from click.testing import CliRunner from pdf_tools import companion, ollama, scan_index from pdf_tools.extract_scan import to_plain_markdown from pdf_tools.scan_config import OllamaHost, load_config from pdf_tools.scan_stats import read_log, summarize HOST = OllamaHost("local", "http://localhost:11434", "gemma4:31b", num_ctx=8192) KINDS = ["calculation", "idea", "meeting notes", "other", "blank"] def answer(**over): a = {"summary": "Estimate of a drag force.", "key_points": ["F = 6 pi eta a U"], "kind": "calculation", "topics": ["Stokes drag"], "references": [], "event": "", "event_date": "", "people": [], "continues_from_previous": False, "continues_to_next": True} return {**a, **over} def make_scan(folder, pages=("first page", "second page", "third page")): folder.mkdir(parents=True, exist_ok=True) pdf = folder / "2009 notebook.pdf" doc = pdfium.PdfDocument.new() for _ in pages: doc.new_page(200, 260) doc.save(pdf) md = pdf.with_suffix(".md") md.write_text(to_plain_markdown(list(pages), pdf.stem, "ollama/localhost/gemma4:31b")) return pdf @pytest.fixture def env(tmp_path, monkeypatch): cfg = tmp_path / "scan-notes.toml" cfg.write_text(f""" names = ["Global Person"] stats_log = "{tmp_path / "stats.jsonl"}" [ollama] max_output_tokens = 2048 [[ollama.hosts]] name = "local" url = "http://localhost:11434" model = "gemma4:31b" num_ctx = 8192 [index] names = ["Board Member"] prompt_extra = "Index-only instruction." """) monkeypatch.setenv("PDF_TOOLS_SCAN_CONFIG", str(cfg)) return tmp_path def fake(monkeypatch, replies=None): """Ollama with gemma4:31b; chat replies come from the list, then answer().""" replies = list(replies or []) def _fake(method, url, payload, timeout): base, _, path = url.partition("/api/") if path == "tags": return {"models": [{"name": "gemma4:31b"}]} if path == "ps": return {"models": [{"name": "gemma4:31b", "size": 20, "size_vram": 20, "context_length": 8192}]} _fake.payloads.append(payload) r = replies.pop(0) if replies else answer() if isinstance(r, Exception): raise r if isinstance(r, dict) and "message" in r: return r return {"message": {"content": r if isinstance(r, str) else json.dumps(r)}, "prompt_eval_count": 2000, "eval_count": 300, "done_reason": "stop", "total_duration": 9_000_000_000} _fake.payloads = [] monkeypatch.setattr(ollama, "_http_json", _fake) return _fake def run(*args): return CliRunner().invoke(scan_index.main, [str(a) for a in args]) def test_reads_plain_and_companion_transcripts(tmp_path): plain = tmp_path / "a.md" plain.write_text(to_plain_markdown(["one", "has ``` inside", ""], "a", "ollama/x/m")) pages, by = scan_index.read_transcript(plain) assert pages == {1: "one", 2: "has ``` inside", 3: ""} and by == "ollama/x/m" comp = tmp_path / "b.md" comp.write_text("---\ntranscribed_by: ollama/y/m\n---\n\n## Notes\n\n" + companion.draft_block(["alpha", "beta"]) + "\n") pages, by = scan_index.read_transcript(comp) assert pages == {1: "alpha", 2: "beta"} and by == "ollama/y/m" def test_prompt_has_kinds_note_extras_and_names(): p = scan_index.build_index_prompt(KINDS, ["Pat Doe"], ["Extra rule."]) assert '"calculation" for derivations' in p assert p.index("Extra rule.") < p.index(ollama.NAMES_PROMPT) < p.index("- Pat Doe") assert '"calculation"' not in scan_index.build_index_prompt(["idea", "other"], [], []) def test_page_prompt_marks_this_page_and_neighbors(): pages = {1: "p1", 2: "p2" * 50, 3: "p3"} p = scan_index.page_prompt("BASE", pages, 1, chars=10) assert "Previous page transcript: (none)" in p assert "THIS page's draft transcript:\n<<<\np1\n>>>" in p p = scan_index.page_prompt("BASE", pages, 3, chars=10) assert "<<<\n" + "p2" * 5 + "\n>>>" in p and "Next page transcript: (none)" in p def test_clean_coerces_types_and_unknown_kind(): m = scan_index.clean({"summary": None, "kind": "derivation", "topics": "one", "people": ["", " A "], "continues_to_next": "yes"}, KINDS) assert m["summary"] == "" and m["kind"] == "other" assert m["topics"] == ["one"] and m["people"] == ["A"] assert m["continues_to_next"] is False and m["references"] == [] def test_render_page_front_matter_and_sections(): meta = scan_index.clean(answer(references=['Smith, "x", J. Fluid Mech. 1 (2001)'], continues_from_previous=True), KINDS) text = scan_index.render_page("nb", "nb.pdf", 1, 3, meta, "a ``` b", "ollama/h/m", "ollama/h/t", "2026-11-02") assert text.startswith('---\nnotebook: "nb"\nscan: "nb.pdf"\npage: 1\nkind: "calculation"') assert 'references: ["Smith, \\"x\\", J. Fluid Mech. 1 (2001)"]' in text assert "continues_from: null\ncontinues_to: 2\n" in text # page 1 has no previous assert "transcribed_by: ollama/h/t" in text assert scan_index.SUMMARY_HEADING + "\n\nEstimate of a drag force." in text assert "## Key points\n\n- F = 6 pi eta a U" in text assert "````text\na ``` b\n````" in text def test_cli_writes_one_file_per_page(env, monkeypatch): f = fake(monkeypatch) pdf = make_scan(env / "nb") r = run(pdf) assert r.exit_code == 0, r.output out = env / "nb" / "2009 notebook pages" files = sorted(p.name for p in out.iterdir()) assert files == [f"2009 notebook p00{n}.md" for n in (1, 2, 3)] p2 = (out / "2009 notebook p002.md").read_text() assert "page: 2\n" in p2 and "continues_to: 3" in p2 and "second page" in p2 assert "indexed_by: ollama/localhost/gemma4:31b" in p2 assert "continues_to: null" in (out / "2009 notebook p003.md").read_text() req = f.payloads[1] assert req["format"]["properties"]["kind"]["enum"][0] == "calculation" assert req["options"] == {"temperature": 0, "num_ctx": 8192, "num_predict": 1536} prompt = req["messages"][0]["content"] assert "- Global Person" in prompt and "- Board Member" in prompt assert "Index-only instruction." in prompt assert "<<<\nsecond page\n>>>" in prompt and "<<<\nfirst page\n>>>" in prompt log = read_log(env / "stats.jsonl") assert [x["task"] for x in log] == ["index"] * 3 and log[0]["kind"] == "calculation" assert "summary" not in json.dumps(log) # no generated text in the log assert "index: gemma4:31b on localhost" in summarize(log) def test_existing_pages_are_skipped_unless_forced(env, monkeypatch): f = fake(monkeypatch) pdf = make_scan(env / "nb") out = env / "nb" / "2009 notebook pages" out.mkdir() (out / "2009 notebook p001.md").write_text("kept") assert run(pdf).exit_code == 0 assert len(f.payloads) == 2 and (out / "2009 notebook p001.md").read_text() == "kept" r = run(pdf) assert r.exit_code == 0 and "already indexed" in r.stderr and len(f.payloads) == 2 assert run("--force", pdf).exit_code == 0 and len(f.payloads) == 5 def test_bad_json_is_retried(env, monkeypatch): f = fake(monkeypatch, ["not json", {"summary": "missing the other keys"}]) pdf = make_scan(env / "nb", pages=("only",)) assert run(pdf).exit_code == 0 assert len(f.payloads) == 3 log = read_log(env / "stats.jsonl") assert log[0]["retries"] == 2 and log[0]["retry_reasons"] == ["bad_json", "bad_json"] def test_failed_page_reported_and_others_written(env, monkeypatch): def err(): return urllib.error.HTTPError("u", 500, "x", {}, io.BytesIO(b'{"error":"token repeat limit"}')) fake(monkeypatch, [err(), err(), err()]) pdf = make_scan(env / "nb", pages=("a", "b")) r = run(pdf) assert r.exit_code == scan_index.EXIT_TRANSCRIBE_FAILED assert "1 page(s) failed (1)" in r.stderr out = env / "nb" / "2009 notebook pages" assert [p.name for p in out.iterdir()] == ["2009 notebook p002.md"] def test_page_count_mismatch_refused(env, monkeypatch): fake(monkeypatch) pdf = make_scan(env / "nb") pdf.with_suffix(".md").write_text(to_plain_markdown(["a"], "x", "y")) r = run(pdf) assert r.exit_code == 2 and "1 transcribed pages" in r.stderr def test_missing_transcript_refused(env, monkeypatch): fake(monkeypatch) pdf = make_scan(env / "nb") pdf.with_suffix(".md").unlink() r = run(pdf) assert r.exit_code == 2 and "run scan-notes-md first" in r.stderr def test_no_host_exits_without_cloud(env, monkeypatch): def down(*a): raise urllib.error.URLError("refused") monkeypatch.setattr(ollama, "_http_json", down) pdf = make_scan(env / "nb") r = run(pdf) assert r.exit_code == scan_index.EXIT_NO_HOST assert "Not falling back to a cloud service" in r.stderr def test_index_config_rejects_unknown_keys(tmp_path, capsys): cfg = tmp_path / "c.toml" cfg.write_text("[index]\nkind = ['x']\n") with pytest.raises(SystemExit): load_config(cfg) assert "unknown key(s) in [index]: kind" in capsys.readouterr().err cfg.write_text("[index]\nkinds = []\n") with pytest.raises(SystemExit): load_config(cfg)