Initial public release of pdf-tools
Command-line tools for marked-up PDFs and scanned handwritten notes: pdf-scrub, pdf-notes-md, scan-notes-md (Ollama by default, Claude on request), scan-notes-index (per-page search metadata), and scan-notes-stats. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011ZSvbNaLhgFuTm7wmkgbWs
This commit is contained in:
commit
e464871b69
29 changed files with 5296 additions and 0 deletions
36
tests/test_stats.py
Normal file
36
tests/test_stats.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
"""The scan-notes-stats summary."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pdf_tools.scan_stats import StatsLog, read_log, summarize
|
||||
|
||||
|
||||
def rec(ctx, run="r1", reason="stop", label="nb.pdf p.1"):
|
||||
return {"run": run, "host": "gpu-box", "model": "gemma4:31b",
|
||||
"loaded_num_ctx": 32768, "label": label, "image_px": [2000, 1500],
|
||||
"prompt_tokens": ctx - 300, "output_tokens": 300, "context_tokens": ctx,
|
||||
"done_reason": reason, "total_s": 10.0, "gpu_fraction": 0.94,
|
||||
"vram_bytes": 19.7e9, "loaded_bytes": 21e9}
|
||||
|
||||
|
||||
def test_threshold_counts_and_extremes():
|
||||
rs = [rec(1500), rec(3000), rec(5000), rec(9000, reason="length", label="big p.9")]
|
||||
out = summarize(rs, thresholds=(4096, 8192))
|
||||
assert "4 pages, 1 run(s)" in out
|
||||
assert "> 4096: 2 (50.0%)" in out and "> 8192: 1 (25.0%)" in out
|
||||
assert "length limit: 1 page(s)" in out
|
||||
assert "model looped: 0 page(s) retried, 0 left as placeholders" in out
|
||||
assert "model in GPU memory: 94-94%" in out
|
||||
assert out.index("big p.9") < out.index("nb.pdf p.1")
|
||||
|
||||
|
||||
def test_log_roundtrip(tmp_path):
|
||||
log = StatsLog(tmp_path / "sub" / "s.jsonl")
|
||||
log.write(rec(100))
|
||||
log.write(rec(200))
|
||||
(tmp_path / "sub" / "s.jsonl").open("a").write("not json\n")
|
||||
assert [r["context_tokens"] for r in read_log(tmp_path / "sub" / "s.jsonl")] == [100, 200]
|
||||
StatsLog(None).write(rec(1)) # disabled log is a no-op
|
||||
|
||||
|
||||
def test_empty():
|
||||
assert summarize([]) == "no pages logged yet"
|
||||
Loading…
Add table
Add a link
Reference in a new issue