Initial public release of pdf-tools
Command-line tools for marked-up PDFs and scanned handwritten notes: pdf-scrub, pdf-notes-md, scan-notes-md (Ollama by default, Claude on request), scan-notes-index (per-page search metadata), and scan-notes-stats. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011ZSvbNaLhgFuTm7wmkgbWs
This commit is contained in:
commit
e464871b69
29 changed files with 5296 additions and 0 deletions
240
tests/test_index.py
Normal file
240
tests/test_index.py
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
"""scan-notes-index: transcript parsing, page files, and the CLI with Ollama faked."""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import urllib.error
|
||||
|
||||
import pypdfium2 as pdfium
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from pdf_tools import companion, ollama, scan_index
|
||||
from pdf_tools.extract_scan import to_plain_markdown
|
||||
from pdf_tools.scan_config import OllamaHost, load_config
|
||||
from pdf_tools.scan_stats import read_log, summarize
|
||||
|
||||
HOST = OllamaHost("local", "http://localhost:11434", "gemma4:31b", num_ctx=8192)
|
||||
KINDS = ["calculation", "idea", "meeting notes", "other", "blank"]
|
||||
|
||||
|
||||
def answer(**over):
|
||||
a = {"summary": "Estimate of a drag force.", "key_points": ["F = 6 pi eta a U"],
|
||||
"kind": "calculation", "topics": ["Stokes drag"], "references": [],
|
||||
"event": "", "event_date": "", "people": [],
|
||||
"continues_from_previous": False, "continues_to_next": True}
|
||||
return {**a, **over}
|
||||
|
||||
|
||||
def make_scan(folder, pages=("first page", "second page", "third page")):
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
pdf = folder / "2009 notebook.pdf"
|
||||
doc = pdfium.PdfDocument.new()
|
||||
for _ in pages:
|
||||
doc.new_page(200, 260)
|
||||
doc.save(pdf)
|
||||
md = pdf.with_suffix(".md")
|
||||
md.write_text(to_plain_markdown(list(pages), pdf.stem, "ollama/localhost/gemma4:31b"))
|
||||
return pdf
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def env(tmp_path, monkeypatch):
|
||||
cfg = tmp_path / "scan-notes.toml"
|
||||
cfg.write_text(f"""
|
||||
names = ["Global Person"]
|
||||
stats_log = "{tmp_path / "stats.jsonl"}"
|
||||
|
||||
[ollama]
|
||||
max_output_tokens = 2048
|
||||
|
||||
[[ollama.hosts]]
|
||||
name = "local"
|
||||
url = "http://localhost:11434"
|
||||
model = "gemma4:31b"
|
||||
num_ctx = 8192
|
||||
|
||||
[index]
|
||||
names = ["Board Member"]
|
||||
prompt_extra = "Index-only instruction."
|
||||
""")
|
||||
monkeypatch.setenv("PDF_TOOLS_SCAN_CONFIG", str(cfg))
|
||||
return tmp_path
|
||||
|
||||
|
||||
def fake(monkeypatch, replies=None):
|
||||
"""Ollama with gemma4:31b; chat replies come from the list, then answer()."""
|
||||
replies = list(replies or [])
|
||||
|
||||
def _fake(method, url, payload, timeout):
|
||||
base, _, path = url.partition("/api/")
|
||||
if path == "tags":
|
||||
return {"models": [{"name": "gemma4:31b"}]}
|
||||
if path == "ps":
|
||||
return {"models": [{"name": "gemma4:31b", "size": 20, "size_vram": 20,
|
||||
"context_length": 8192}]}
|
||||
_fake.payloads.append(payload)
|
||||
r = replies.pop(0) if replies else answer()
|
||||
if isinstance(r, Exception):
|
||||
raise r
|
||||
if isinstance(r, dict) and "message" in r:
|
||||
return r
|
||||
return {"message": {"content": r if isinstance(r, str) else json.dumps(r)},
|
||||
"prompt_eval_count": 2000, "eval_count": 300, "done_reason": "stop",
|
||||
"total_duration": 9_000_000_000}
|
||||
_fake.payloads = []
|
||||
monkeypatch.setattr(ollama, "_http_json", _fake)
|
||||
return _fake
|
||||
|
||||
|
||||
def run(*args):
|
||||
return CliRunner().invoke(scan_index.main, [str(a) for a in args])
|
||||
|
||||
|
||||
def test_reads_plain_and_companion_transcripts(tmp_path):
|
||||
plain = tmp_path / "a.md"
|
||||
plain.write_text(to_plain_markdown(["one", "has ``` inside", ""], "a", "ollama/x/m"))
|
||||
pages, by = scan_index.read_transcript(plain)
|
||||
assert pages == {1: "one", 2: "has ``` inside", 3: ""} and by == "ollama/x/m"
|
||||
|
||||
comp = tmp_path / "b.md"
|
||||
comp.write_text("---\ntranscribed_by: ollama/y/m\n---\n\n## Notes\n\n"
|
||||
+ companion.draft_block(["alpha", "beta"]) + "\n")
|
||||
pages, by = scan_index.read_transcript(comp)
|
||||
assert pages == {1: "alpha", 2: "beta"} and by == "ollama/y/m"
|
||||
|
||||
|
||||
def test_prompt_has_kinds_note_extras_and_names():
|
||||
p = scan_index.build_index_prompt(KINDS, ["Pat Doe"], ["Extra rule."])
|
||||
assert '"calculation" for derivations' in p
|
||||
assert p.index("Extra rule.") < p.index(ollama.NAMES_PROMPT) < p.index("- Pat Doe")
|
||||
assert '"calculation"' not in scan_index.build_index_prompt(["idea", "other"], [], [])
|
||||
|
||||
|
||||
def test_page_prompt_marks_this_page_and_neighbors():
|
||||
pages = {1: "p1", 2: "p2" * 50, 3: "p3"}
|
||||
p = scan_index.page_prompt("BASE", pages, 1, chars=10)
|
||||
assert "Previous page transcript: (none)" in p
|
||||
assert "THIS page's draft transcript:\n<<<\np1\n>>>" in p
|
||||
p = scan_index.page_prompt("BASE", pages, 3, chars=10)
|
||||
assert "<<<\n" + "p2" * 5 + "\n>>>" in p and "Next page transcript: (none)" in p
|
||||
|
||||
|
||||
def test_clean_coerces_types_and_unknown_kind():
|
||||
m = scan_index.clean({"summary": None, "kind": "derivation", "topics": "one",
|
||||
"people": ["", " A "], "continues_to_next": "yes"}, KINDS)
|
||||
assert m["summary"] == "" and m["kind"] == "other"
|
||||
assert m["topics"] == ["one"] and m["people"] == ["A"]
|
||||
assert m["continues_to_next"] is False and m["references"] == []
|
||||
|
||||
|
||||
def test_render_page_front_matter_and_sections():
|
||||
meta = scan_index.clean(answer(references=['Smith, "x", J. Fluid Mech. 1 (2001)'],
|
||||
continues_from_previous=True), KINDS)
|
||||
text = scan_index.render_page("nb", "nb.pdf", 1, 3, meta, "a ``` b", "ollama/h/m",
|
||||
"ollama/h/t", "2026-11-02")
|
||||
assert text.startswith('---\nnotebook: "nb"\nscan: "nb.pdf"\npage: 1\nkind: "calculation"')
|
||||
assert 'references: ["Smith, \\"x\\", J. Fluid Mech. 1 (2001)"]' in text
|
||||
assert "continues_from: null\ncontinues_to: 2\n" in text # page 1 has no previous
|
||||
assert "transcribed_by: ollama/h/t" in text
|
||||
assert scan_index.SUMMARY_HEADING + "\n\nEstimate of a drag force." in text
|
||||
assert "## Key points\n\n- F = 6 pi eta a U" in text
|
||||
assert "````text\na ``` b\n````" in text
|
||||
|
||||
|
||||
def test_cli_writes_one_file_per_page(env, monkeypatch):
|
||||
f = fake(monkeypatch)
|
||||
pdf = make_scan(env / "nb")
|
||||
r = run(pdf)
|
||||
assert r.exit_code == 0, r.output
|
||||
out = env / "nb" / "2009 notebook pages"
|
||||
files = sorted(p.name for p in out.iterdir())
|
||||
assert files == [f"2009 notebook p00{n}.md" for n in (1, 2, 3)]
|
||||
p2 = (out / "2009 notebook p002.md").read_text()
|
||||
assert "page: 2\n" in p2 and "continues_to: 3" in p2 and "second page" in p2
|
||||
assert "indexed_by: ollama/localhost/gemma4:31b" in p2
|
||||
assert "continues_to: null" in (out / "2009 notebook p003.md").read_text()
|
||||
|
||||
req = f.payloads[1]
|
||||
assert req["format"]["properties"]["kind"]["enum"][0] == "calculation"
|
||||
assert req["options"] == {"temperature": 0, "num_ctx": 8192, "num_predict": 1536}
|
||||
prompt = req["messages"][0]["content"]
|
||||
assert "- Global Person" in prompt and "- Board Member" in prompt
|
||||
assert "Index-only instruction." in prompt
|
||||
assert "<<<\nsecond page\n>>>" in prompt and "<<<\nfirst page\n>>>" in prompt
|
||||
|
||||
log = read_log(env / "stats.jsonl")
|
||||
assert [x["task"] for x in log] == ["index"] * 3 and log[0]["kind"] == "calculation"
|
||||
assert "summary" not in json.dumps(log) # no generated text in the log
|
||||
assert "index: gemma4:31b on localhost" in summarize(log)
|
||||
|
||||
|
||||
def test_existing_pages_are_skipped_unless_forced(env, monkeypatch):
|
||||
f = fake(monkeypatch)
|
||||
pdf = make_scan(env / "nb")
|
||||
out = env / "nb" / "2009 notebook pages"
|
||||
out.mkdir()
|
||||
(out / "2009 notebook p001.md").write_text("kept")
|
||||
assert run(pdf).exit_code == 0
|
||||
assert len(f.payloads) == 2 and (out / "2009 notebook p001.md").read_text() == "kept"
|
||||
r = run(pdf)
|
||||
assert r.exit_code == 0 and "already indexed" in r.stderr and len(f.payloads) == 2
|
||||
assert run("--force", pdf).exit_code == 0 and len(f.payloads) == 5
|
||||
|
||||
|
||||
def test_bad_json_is_retried(env, monkeypatch):
|
||||
f = fake(monkeypatch, ["not json", {"summary": "missing the other keys"}])
|
||||
pdf = make_scan(env / "nb", pages=("only",))
|
||||
assert run(pdf).exit_code == 0
|
||||
assert len(f.payloads) == 3
|
||||
log = read_log(env / "stats.jsonl")
|
||||
assert log[0]["retries"] == 2 and log[0]["retry_reasons"] == ["bad_json", "bad_json"]
|
||||
|
||||
|
||||
def test_failed_page_reported_and_others_written(env, monkeypatch):
|
||||
def err():
|
||||
return urllib.error.HTTPError("u", 500, "x", {}, io.BytesIO(b'{"error":"token repeat limit"}'))
|
||||
fake(monkeypatch, [err(), err(), err()])
|
||||
pdf = make_scan(env / "nb", pages=("a", "b"))
|
||||
r = run(pdf)
|
||||
assert r.exit_code == scan_index.EXIT_TRANSCRIBE_FAILED
|
||||
assert "1 page(s) failed (1)" in r.stderr
|
||||
out = env / "nb" / "2009 notebook pages"
|
||||
assert [p.name for p in out.iterdir()] == ["2009 notebook p002.md"]
|
||||
|
||||
|
||||
def test_page_count_mismatch_refused(env, monkeypatch):
|
||||
fake(monkeypatch)
|
||||
pdf = make_scan(env / "nb")
|
||||
pdf.with_suffix(".md").write_text(to_plain_markdown(["a"], "x", "y"))
|
||||
r = run(pdf)
|
||||
assert r.exit_code == 2 and "1 transcribed pages" in r.stderr
|
||||
|
||||
|
||||
def test_missing_transcript_refused(env, monkeypatch):
|
||||
fake(monkeypatch)
|
||||
pdf = make_scan(env / "nb")
|
||||
pdf.with_suffix(".md").unlink()
|
||||
r = run(pdf)
|
||||
assert r.exit_code == 2 and "run scan-notes-md first" in r.stderr
|
||||
|
||||
|
||||
def test_no_host_exits_without_cloud(env, monkeypatch):
|
||||
def down(*a):
|
||||
raise urllib.error.URLError("refused")
|
||||
monkeypatch.setattr(ollama, "_http_json", down)
|
||||
pdf = make_scan(env / "nb")
|
||||
r = run(pdf)
|
||||
assert r.exit_code == scan_index.EXIT_NO_HOST
|
||||
assert "Not falling back to a cloud service" in r.stderr
|
||||
|
||||
|
||||
def test_index_config_rejects_unknown_keys(tmp_path, capsys):
|
||||
cfg = tmp_path / "c.toml"
|
||||
cfg.write_text("[index]\nkind = ['x']\n")
|
||||
with pytest.raises(SystemExit):
|
||||
load_config(cfg)
|
||||
assert "unknown key(s) in [index]: kind" in capsys.readouterr().err
|
||||
cfg.write_text("[index]\nkinds = []\n")
|
||||
with pytest.raises(SystemExit):
|
||||
load_config(cfg)
|
||||
Loading…
Add table
Add a link
Reference in a new issue