Command-line tools for marked-up PDFs and scanned handwritten notes: pdf-scrub, pdf-notes-md, scan-notes-md (Ollama by default, Claude on request), scan-notes-index (per-page search metadata), and scan-notes-stats. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011ZSvbNaLhgFuTm7wmkgbWs
240 lines
9.3 KiB
Python
240 lines
9.3 KiB
Python
"""scan-notes-index: transcript parsing, page files, and the CLI with Ollama faked."""
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import urllib.error
|
|
|
|
import pypdfium2 as pdfium
|
|
import pytest
|
|
from click.testing import CliRunner
|
|
|
|
from pdf_tools import companion, ollama, scan_index
|
|
from pdf_tools.extract_scan import to_plain_markdown
|
|
from pdf_tools.scan_config import OllamaHost, load_config
|
|
from pdf_tools.scan_stats import read_log, summarize
|
|
|
|
HOST = OllamaHost("local", "http://localhost:11434", "gemma4:31b", num_ctx=8192)
|
|
KINDS = ["calculation", "idea", "meeting notes", "other", "blank"]
|
|
|
|
|
|
def answer(**over):
|
|
a = {"summary": "Estimate of a drag force.", "key_points": ["F = 6 pi eta a U"],
|
|
"kind": "calculation", "topics": ["Stokes drag"], "references": [],
|
|
"event": "", "event_date": "", "people": [],
|
|
"continues_from_previous": False, "continues_to_next": True}
|
|
return {**a, **over}
|
|
|
|
|
|
def make_scan(folder, pages=("first page", "second page", "third page")):
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
pdf = folder / "2009 notebook.pdf"
|
|
doc = pdfium.PdfDocument.new()
|
|
for _ in pages:
|
|
doc.new_page(200, 260)
|
|
doc.save(pdf)
|
|
md = pdf.with_suffix(".md")
|
|
md.write_text(to_plain_markdown(list(pages), pdf.stem, "ollama/localhost/gemma4:31b"))
|
|
return pdf
|
|
|
|
|
|
@pytest.fixture
|
|
def env(tmp_path, monkeypatch):
|
|
cfg = tmp_path / "scan-notes.toml"
|
|
cfg.write_text(f"""
|
|
names = ["Global Person"]
|
|
stats_log = "{tmp_path / "stats.jsonl"}"
|
|
|
|
[ollama]
|
|
max_output_tokens = 2048
|
|
|
|
[[ollama.hosts]]
|
|
name = "local"
|
|
url = "http://localhost:11434"
|
|
model = "gemma4:31b"
|
|
num_ctx = 8192
|
|
|
|
[index]
|
|
names = ["Board Member"]
|
|
prompt_extra = "Index-only instruction."
|
|
""")
|
|
monkeypatch.setenv("PDF_TOOLS_SCAN_CONFIG", str(cfg))
|
|
return tmp_path
|
|
|
|
|
|
def fake(monkeypatch, replies=None):
|
|
"""Ollama with gemma4:31b; chat replies come from the list, then answer()."""
|
|
replies = list(replies or [])
|
|
|
|
def _fake(method, url, payload, timeout):
|
|
base, _, path = url.partition("/api/")
|
|
if path == "tags":
|
|
return {"models": [{"name": "gemma4:31b"}]}
|
|
if path == "ps":
|
|
return {"models": [{"name": "gemma4:31b", "size": 20, "size_vram": 20,
|
|
"context_length": 8192}]}
|
|
_fake.payloads.append(payload)
|
|
r = replies.pop(0) if replies else answer()
|
|
if isinstance(r, Exception):
|
|
raise r
|
|
if isinstance(r, dict) and "message" in r:
|
|
return r
|
|
return {"message": {"content": r if isinstance(r, str) else json.dumps(r)},
|
|
"prompt_eval_count": 2000, "eval_count": 300, "done_reason": "stop",
|
|
"total_duration": 9_000_000_000}
|
|
_fake.payloads = []
|
|
monkeypatch.setattr(ollama, "_http_json", _fake)
|
|
return _fake
|
|
|
|
|
|
def run(*args):
|
|
return CliRunner().invoke(scan_index.main, [str(a) for a in args])
|
|
|
|
|
|
def test_reads_plain_and_companion_transcripts(tmp_path):
|
|
plain = tmp_path / "a.md"
|
|
plain.write_text(to_plain_markdown(["one", "has ``` inside", ""], "a", "ollama/x/m"))
|
|
pages, by = scan_index.read_transcript(plain)
|
|
assert pages == {1: "one", 2: "has ``` inside", 3: ""} and by == "ollama/x/m"
|
|
|
|
comp = tmp_path / "b.md"
|
|
comp.write_text("---\ntranscribed_by: ollama/y/m\n---\n\n## Notes\n\n"
|
|
+ companion.draft_block(["alpha", "beta"]) + "\n")
|
|
pages, by = scan_index.read_transcript(comp)
|
|
assert pages == {1: "alpha", 2: "beta"} and by == "ollama/y/m"
|
|
|
|
|
|
def test_prompt_has_kinds_note_extras_and_names():
|
|
p = scan_index.build_index_prompt(KINDS, ["Pat Doe"], ["Extra rule."])
|
|
assert '"calculation" for derivations' in p
|
|
assert p.index("Extra rule.") < p.index(ollama.NAMES_PROMPT) < p.index("- Pat Doe")
|
|
assert '"calculation"' not in scan_index.build_index_prompt(["idea", "other"], [], [])
|
|
|
|
|
|
def test_page_prompt_marks_this_page_and_neighbors():
|
|
pages = {1: "p1", 2: "p2" * 50, 3: "p3"}
|
|
p = scan_index.page_prompt("BASE", pages, 1, chars=10)
|
|
assert "Previous page transcript: (none)" in p
|
|
assert "THIS page's draft transcript:\n<<<\np1\n>>>" in p
|
|
p = scan_index.page_prompt("BASE", pages, 3, chars=10)
|
|
assert "<<<\n" + "p2" * 5 + "\n>>>" in p and "Next page transcript: (none)" in p
|
|
|
|
|
|
def test_clean_coerces_types_and_unknown_kind():
|
|
m = scan_index.clean({"summary": None, "kind": "derivation", "topics": "one",
|
|
"people": ["", " A "], "continues_to_next": "yes"}, KINDS)
|
|
assert m["summary"] == "" and m["kind"] == "other"
|
|
assert m["topics"] == ["one"] and m["people"] == ["A"]
|
|
assert m["continues_to_next"] is False and m["references"] == []
|
|
|
|
|
|
def test_render_page_front_matter_and_sections():
|
|
meta = scan_index.clean(answer(references=['Smith, "x", J. Fluid Mech. 1 (2001)'],
|
|
continues_from_previous=True), KINDS)
|
|
text = scan_index.render_page("nb", "nb.pdf", 1, 3, meta, "a ``` b", "ollama/h/m",
|
|
"ollama/h/t", "2026-11-02")
|
|
assert text.startswith('---\nnotebook: "nb"\nscan: "nb.pdf"\npage: 1\nkind: "calculation"')
|
|
assert 'references: ["Smith, \\"x\\", J. Fluid Mech. 1 (2001)"]' in text
|
|
assert "continues_from: null\ncontinues_to: 2\n" in text # page 1 has no previous
|
|
assert "transcribed_by: ollama/h/t" in text
|
|
assert scan_index.SUMMARY_HEADING + "\n\nEstimate of a drag force." in text
|
|
assert "## Key points\n\n- F = 6 pi eta a U" in text
|
|
assert "````text\na ``` b\n````" in text
|
|
|
|
|
|
def test_cli_writes_one_file_per_page(env, monkeypatch):
|
|
f = fake(monkeypatch)
|
|
pdf = make_scan(env / "nb")
|
|
r = run(pdf)
|
|
assert r.exit_code == 0, r.output
|
|
out = env / "nb" / "2009 notebook pages"
|
|
files = sorted(p.name for p in out.iterdir())
|
|
assert files == [f"2009 notebook p00{n}.md" for n in (1, 2, 3)]
|
|
p2 = (out / "2009 notebook p002.md").read_text()
|
|
assert "page: 2\n" in p2 and "continues_to: 3" in p2 and "second page" in p2
|
|
assert "indexed_by: ollama/localhost/gemma4:31b" in p2
|
|
assert "continues_to: null" in (out / "2009 notebook p003.md").read_text()
|
|
|
|
req = f.payloads[1]
|
|
assert req["format"]["properties"]["kind"]["enum"][0] == "calculation"
|
|
assert req["options"] == {"temperature": 0, "num_ctx": 8192, "num_predict": 1536}
|
|
prompt = req["messages"][0]["content"]
|
|
assert "- Global Person" in prompt and "- Board Member" in prompt
|
|
assert "Index-only instruction." in prompt
|
|
assert "<<<\nsecond page\n>>>" in prompt and "<<<\nfirst page\n>>>" in prompt
|
|
|
|
log = read_log(env / "stats.jsonl")
|
|
assert [x["task"] for x in log] == ["index"] * 3 and log[0]["kind"] == "calculation"
|
|
assert "summary" not in json.dumps(log) # no generated text in the log
|
|
assert "index: gemma4:31b on localhost" in summarize(log)
|
|
|
|
|
|
def test_existing_pages_are_skipped_unless_forced(env, monkeypatch):
|
|
f = fake(monkeypatch)
|
|
pdf = make_scan(env / "nb")
|
|
out = env / "nb" / "2009 notebook pages"
|
|
out.mkdir()
|
|
(out / "2009 notebook p001.md").write_text("kept")
|
|
assert run(pdf).exit_code == 0
|
|
assert len(f.payloads) == 2 and (out / "2009 notebook p001.md").read_text() == "kept"
|
|
r = run(pdf)
|
|
assert r.exit_code == 0 and "already indexed" in r.stderr and len(f.payloads) == 2
|
|
assert run("--force", pdf).exit_code == 0 and len(f.payloads) == 5
|
|
|
|
|
|
def test_bad_json_is_retried(env, monkeypatch):
|
|
f = fake(monkeypatch, ["not json", {"summary": "missing the other keys"}])
|
|
pdf = make_scan(env / "nb", pages=("only",))
|
|
assert run(pdf).exit_code == 0
|
|
assert len(f.payloads) == 3
|
|
log = read_log(env / "stats.jsonl")
|
|
assert log[0]["retries"] == 2 and log[0]["retry_reasons"] == ["bad_json", "bad_json"]
|
|
|
|
|
|
def test_failed_page_reported_and_others_written(env, monkeypatch):
|
|
def err():
|
|
return urllib.error.HTTPError("u", 500, "x", {}, io.BytesIO(b'{"error":"token repeat limit"}'))
|
|
fake(monkeypatch, [err(), err(), err()])
|
|
pdf = make_scan(env / "nb", pages=("a", "b"))
|
|
r = run(pdf)
|
|
assert r.exit_code == scan_index.EXIT_TRANSCRIBE_FAILED
|
|
assert "1 page(s) failed (1)" in r.stderr
|
|
out = env / "nb" / "2009 notebook pages"
|
|
assert [p.name for p in out.iterdir()] == ["2009 notebook p002.md"]
|
|
|
|
|
|
def test_page_count_mismatch_refused(env, monkeypatch):
|
|
fake(monkeypatch)
|
|
pdf = make_scan(env / "nb")
|
|
pdf.with_suffix(".md").write_text(to_plain_markdown(["a"], "x", "y"))
|
|
r = run(pdf)
|
|
assert r.exit_code == 2 and "1 transcribed pages" in r.stderr
|
|
|
|
|
|
def test_missing_transcript_refused(env, monkeypatch):
|
|
fake(monkeypatch)
|
|
pdf = make_scan(env / "nb")
|
|
pdf.with_suffix(".md").unlink()
|
|
r = run(pdf)
|
|
assert r.exit_code == 2 and "run scan-notes-md first" in r.stderr
|
|
|
|
|
|
def test_no_host_exits_without_cloud(env, monkeypatch):
|
|
def down(*a):
|
|
raise urllib.error.URLError("refused")
|
|
monkeypatch.setattr(ollama, "_http_json", down)
|
|
pdf = make_scan(env / "nb")
|
|
r = run(pdf)
|
|
assert r.exit_code == scan_index.EXIT_NO_HOST
|
|
assert "Not falling back to a cloud service" in r.stderr
|
|
|
|
|
|
def test_index_config_rejects_unknown_keys(tmp_path, capsys):
|
|
cfg = tmp_path / "c.toml"
|
|
cfg.write_text("[index]\nkind = ['x']\n")
|
|
with pytest.raises(SystemExit):
|
|
load_config(cfg)
|
|
assert "unknown key(s) in [index]: kind" in capsys.readouterr().err
|
|
cfg.write_text("[index]\nkinds = []\n")
|
|
with pytest.raises(SystemExit):
|
|
load_config(cfg)
|