pdf-tools/tests/test_index.py
Eric Furst e464871b69 Initial public release of pdf-tools
Command-line tools for marked-up PDFs and scanned handwritten notes:
pdf-scrub, pdf-notes-md, scan-notes-md (Ollama by default, Claude on
request), scan-notes-index (per-page search metadata), and
scan-notes-stats.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011ZSvbNaLhgFuTm7wmkgbWs
2026-11-03 07:25:51 -05:00

240 lines
9.3 KiB
Python

"""scan-notes-index: transcript parsing, page files, and the CLI with Ollama faked."""
from __future__ import annotations
import io
import json
import urllib.error
import pypdfium2 as pdfium
import pytest
from click.testing import CliRunner
from pdf_tools import companion, ollama, scan_index
from pdf_tools.extract_scan import to_plain_markdown
from pdf_tools.scan_config import OllamaHost, load_config
from pdf_tools.scan_stats import read_log, summarize
HOST = OllamaHost("local", "http://localhost:11434", "gemma4:31b", num_ctx=8192)
KINDS = ["calculation", "idea", "meeting notes", "other", "blank"]
def answer(**over):
a = {"summary": "Estimate of a drag force.", "key_points": ["F = 6 pi eta a U"],
"kind": "calculation", "topics": ["Stokes drag"], "references": [],
"event": "", "event_date": "", "people": [],
"continues_from_previous": False, "continues_to_next": True}
return {**a, **over}
def make_scan(folder, pages=("first page", "second page", "third page")):
folder.mkdir(parents=True, exist_ok=True)
pdf = folder / "2009 notebook.pdf"
doc = pdfium.PdfDocument.new()
for _ in pages:
doc.new_page(200, 260)
doc.save(pdf)
md = pdf.with_suffix(".md")
md.write_text(to_plain_markdown(list(pages), pdf.stem, "ollama/localhost/gemma4:31b"))
return pdf
@pytest.fixture
def env(tmp_path, monkeypatch):
cfg = tmp_path / "scan-notes.toml"
cfg.write_text(f"""
names = ["Global Person"]
stats_log = "{tmp_path / "stats.jsonl"}"
[ollama]
max_output_tokens = 2048
[[ollama.hosts]]
name = "local"
url = "http://localhost:11434"
model = "gemma4:31b"
num_ctx = 8192
[index]
names = ["Board Member"]
prompt_extra = "Index-only instruction."
""")
monkeypatch.setenv("PDF_TOOLS_SCAN_CONFIG", str(cfg))
return tmp_path
def fake(monkeypatch, replies=None):
"""Ollama with gemma4:31b; chat replies come from the list, then answer()."""
replies = list(replies or [])
def _fake(method, url, payload, timeout):
base, _, path = url.partition("/api/")
if path == "tags":
return {"models": [{"name": "gemma4:31b"}]}
if path == "ps":
return {"models": [{"name": "gemma4:31b", "size": 20, "size_vram": 20,
"context_length": 8192}]}
_fake.payloads.append(payload)
r = replies.pop(0) if replies else answer()
if isinstance(r, Exception):
raise r
if isinstance(r, dict) and "message" in r:
return r
return {"message": {"content": r if isinstance(r, str) else json.dumps(r)},
"prompt_eval_count": 2000, "eval_count": 300, "done_reason": "stop",
"total_duration": 9_000_000_000}
_fake.payloads = []
monkeypatch.setattr(ollama, "_http_json", _fake)
return _fake
def run(*args):
return CliRunner().invoke(scan_index.main, [str(a) for a in args])
def test_reads_plain_and_companion_transcripts(tmp_path):
plain = tmp_path / "a.md"
plain.write_text(to_plain_markdown(["one", "has ``` inside", ""], "a", "ollama/x/m"))
pages, by = scan_index.read_transcript(plain)
assert pages == {1: "one", 2: "has ``` inside", 3: ""} and by == "ollama/x/m"
comp = tmp_path / "b.md"
comp.write_text("---\ntranscribed_by: ollama/y/m\n---\n\n## Notes\n\n"
+ companion.draft_block(["alpha", "beta"]) + "\n")
pages, by = scan_index.read_transcript(comp)
assert pages == {1: "alpha", 2: "beta"} and by == "ollama/y/m"
def test_prompt_has_kinds_note_extras_and_names():
p = scan_index.build_index_prompt(KINDS, ["Pat Doe"], ["Extra rule."])
assert '"calculation" for derivations' in p
assert p.index("Extra rule.") < p.index(ollama.NAMES_PROMPT) < p.index("- Pat Doe")
assert '"calculation"' not in scan_index.build_index_prompt(["idea", "other"], [], [])
def test_page_prompt_marks_this_page_and_neighbors():
pages = {1: "p1", 2: "p2" * 50, 3: "p3"}
p = scan_index.page_prompt("BASE", pages, 1, chars=10)
assert "Previous page transcript: (none)" in p
assert "THIS page's draft transcript:\n<<<\np1\n>>>" in p
p = scan_index.page_prompt("BASE", pages, 3, chars=10)
assert "<<<\n" + "p2" * 5 + "\n>>>" in p and "Next page transcript: (none)" in p
def test_clean_coerces_types_and_unknown_kind():
m = scan_index.clean({"summary": None, "kind": "derivation", "topics": "one",
"people": ["", " A "], "continues_to_next": "yes"}, KINDS)
assert m["summary"] == "" and m["kind"] == "other"
assert m["topics"] == ["one"] and m["people"] == ["A"]
assert m["continues_to_next"] is False and m["references"] == []
def test_render_page_front_matter_and_sections():
meta = scan_index.clean(answer(references=['Smith, "x", J. Fluid Mech. 1 (2001)'],
continues_from_previous=True), KINDS)
text = scan_index.render_page("nb", "nb.pdf", 1, 3, meta, "a ``` b", "ollama/h/m",
"ollama/h/t", "2026-11-02")
assert text.startswith('---\nnotebook: "nb"\nscan: "nb.pdf"\npage: 1\nkind: "calculation"')
assert 'references: ["Smith, \\"x\\", J. Fluid Mech. 1 (2001)"]' in text
assert "continues_from: null\ncontinues_to: 2\n" in text # page 1 has no previous
assert "transcribed_by: ollama/h/t" in text
assert scan_index.SUMMARY_HEADING + "\n\nEstimate of a drag force." in text
assert "## Key points\n\n- F = 6 pi eta a U" in text
assert "````text\na ``` b\n````" in text
def test_cli_writes_one_file_per_page(env, monkeypatch):
f = fake(monkeypatch)
pdf = make_scan(env / "nb")
r = run(pdf)
assert r.exit_code == 0, r.output
out = env / "nb" / "2009 notebook pages"
files = sorted(p.name for p in out.iterdir())
assert files == [f"2009 notebook p00{n}.md" for n in (1, 2, 3)]
p2 = (out / "2009 notebook p002.md").read_text()
assert "page: 2\n" in p2 and "continues_to: 3" in p2 and "second page" in p2
assert "indexed_by: ollama/localhost/gemma4:31b" in p2
assert "continues_to: null" in (out / "2009 notebook p003.md").read_text()
req = f.payloads[1]
assert req["format"]["properties"]["kind"]["enum"][0] == "calculation"
assert req["options"] == {"temperature": 0, "num_ctx": 8192, "num_predict": 1536}
prompt = req["messages"][0]["content"]
assert "- Global Person" in prompt and "- Board Member" in prompt
assert "Index-only instruction." in prompt
assert "<<<\nsecond page\n>>>" in prompt and "<<<\nfirst page\n>>>" in prompt
log = read_log(env / "stats.jsonl")
assert [x["task"] for x in log] == ["index"] * 3 and log[0]["kind"] == "calculation"
assert "summary" not in json.dumps(log) # no generated text in the log
assert "index: gemma4:31b on localhost" in summarize(log)
def test_existing_pages_are_skipped_unless_forced(env, monkeypatch):
f = fake(monkeypatch)
pdf = make_scan(env / "nb")
out = env / "nb" / "2009 notebook pages"
out.mkdir()
(out / "2009 notebook p001.md").write_text("kept")
assert run(pdf).exit_code == 0
assert len(f.payloads) == 2 and (out / "2009 notebook p001.md").read_text() == "kept"
r = run(pdf)
assert r.exit_code == 0 and "already indexed" in r.stderr and len(f.payloads) == 2
assert run("--force", pdf).exit_code == 0 and len(f.payloads) == 5
def test_bad_json_is_retried(env, monkeypatch):
f = fake(monkeypatch, ["not json", {"summary": "missing the other keys"}])
pdf = make_scan(env / "nb", pages=("only",))
assert run(pdf).exit_code == 0
assert len(f.payloads) == 3
log = read_log(env / "stats.jsonl")
assert log[0]["retries"] == 2 and log[0]["retry_reasons"] == ["bad_json", "bad_json"]
def test_failed_page_reported_and_others_written(env, monkeypatch):
def err():
return urllib.error.HTTPError("u", 500, "x", {}, io.BytesIO(b'{"error":"token repeat limit"}'))
fake(monkeypatch, [err(), err(), err()])
pdf = make_scan(env / "nb", pages=("a", "b"))
r = run(pdf)
assert r.exit_code == scan_index.EXIT_TRANSCRIBE_FAILED
assert "1 page(s) failed (1)" in r.stderr
out = env / "nb" / "2009 notebook pages"
assert [p.name for p in out.iterdir()] == ["2009 notebook p002.md"]
def test_page_count_mismatch_refused(env, monkeypatch):
fake(monkeypatch)
pdf = make_scan(env / "nb")
pdf.with_suffix(".md").write_text(to_plain_markdown(["a"], "x", "y"))
r = run(pdf)
assert r.exit_code == 2 and "1 transcribed pages" in r.stderr
def test_missing_transcript_refused(env, monkeypatch):
fake(monkeypatch)
pdf = make_scan(env / "nb")
pdf.with_suffix(".md").unlink()
r = run(pdf)
assert r.exit_code == 2 and "run scan-notes-md first" in r.stderr
def test_no_host_exits_without_cloud(env, monkeypatch):
def down(*a):
raise urllib.error.URLError("refused")
monkeypatch.setattr(ollama, "_http_json", down)
pdf = make_scan(env / "nb")
r = run(pdf)
assert r.exit_code == scan_index.EXIT_NO_HOST
assert "Not falling back to a cloud service" in r.stderr
def test_index_config_rejects_unknown_keys(tmp_path, capsys):
cfg = tmp_path / "c.toml"
cfg.write_text("[index]\nkind = ['x']\n")
with pytest.raises(SystemExit):
load_config(cfg)
assert "unknown key(s) in [index]: kind" in capsys.readouterr().err
cfg.write_text("[index]\nkinds = []\n")
with pytest.raises(SystemExit):
load_config(cfg)