Command-line tools for marked-up PDFs and scanned handwritten notes: pdf-scrub, pdf-notes-md, scan-notes-md (Ollama by default, Claude on request), scan-notes-index (per-page search metadata), and scan-notes-stats. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011ZSvbNaLhgFuTm7wmkgbWs
439 lines
17 KiB
Python
439 lines
17 KiB
Python
"""Tool 3: extract notes from scanned notebook pages to Markdown.
|
|
|
|
Accepts either:
|
|
- One or more image files (jpg/png/heic/tiff)
|
|
- A PDF (each page is rasterized and processed)
|
|
|
|
Two backends:
|
|
- ollama (default): a vision model on a machine you own, through the Ollama HTTP
|
|
API (see ollama.py). Hosts are tried in the order given in the config
|
|
file (see scan_config.py). There is no fallback to any cloud service.
|
|
- claude: the Anthropic API, used only when asked for with --backend
|
|
claude or the config. Set ANTHROPIC_API_KEY. Model defaults to Claude
|
|
Opus; override with --model.
|
|
|
|
Scans under a confidential folder (declared in the config file; none are
|
|
built in) default to the ollama backend, refuse the claude backend unless
|
|
--allow-cloud is given, and are written to a companion Markdown file shaped
|
|
by that folder's settings (see companion.py).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import dataclasses
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Callable
|
|
from urllib.parse import urlparse
|
|
|
|
import click
|
|
from PIL import Image
|
|
|
|
from . import companion
|
|
from .ollama import (REPEAT_PLACEHOLDER, REPEAT_RETRIES, NoHostAvailable,
|
|
OllamaError, PageOutput, RepetitionError, build_prompt,
|
|
loaded_model, provenance, select_host, transcribe_png)
|
|
from .scan_config import load_config
|
|
from .scan_stats import StatsLog
|
|
|
|
# Anthropic is imported lazily inside _transcribe_image so the rest of
|
|
# the module is usable without the API key set (e.g. for --help), and so
|
|
# the ollama backend never loads it.
|
|
|
|
# Reasonable defaults. The model is one of the few things you might
|
|
# want to tweak per-run as new versions ship. Opus is the default
|
|
# because handwriting transcription is the one place accuracy
|
|
# clearly justifies the cost.
|
|
DEFAULT_MODEL = "claude-opus-5-5"
|
|
SUPPORTED_IMAGE_EXT = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".tif", ".tiff", ".heic"}
|
|
|
|
# Exit codes beyond click's usual 1 and 2.
|
|
EXIT_NO_HOST = 3
|
|
EXIT_TRANSCRIBE_FAILED = 4
|
|
|
|
TRANSCRIBE_PROMPT = """\
|
|
You are transcribing a scanned page from a handwritten research notebook.
|
|
|
|
Output requirements:
|
|
- Reproduce the page content as clean Markdown.
|
|
- Preserve structure: headings, bullet points, indentation, numbered lists.
|
|
- Render equations and chemical formulas in LaTeX. Inline math uses $...$, display math uses $$...$$.
|
|
- Preserve units, subscripts, and Greek letters faithfully.
|
|
- For figures, diagrams, or sketches, insert a single line like:
|
|
`[FIGURE: brief description of what is drawn]`
|
|
- For text you cannot read with confidence, use `[?]` for a single illegible word
|
|
or `[??: best guess]` if you have a tentative reading.
|
|
- Do NOT add commentary, summaries, or explanations. Transcribe only.
|
|
- Do NOT wrap your answer in code fences. Output raw Markdown directly.
|
|
|
|
If the page is blank or contains no notebook content, output exactly:
|
|
[BLANK PAGE]
|
|
"""
|
|
|
|
|
|
@dataclass
|
|
class PageResult:
|
|
index: int # 1-indexed page/image number
|
|
source: str # filename or "page N of source.pdf"
|
|
markdown: str
|
|
|
|
|
|
def _image_to_jpeg_bytes(img: Image.Image, max_side: int = 2000, quality: int = 85) -> bytes:
|
|
"""Downscale + re-encode to keep API payloads reasonable."""
|
|
img = img.convert("RGB")
|
|
w, h = img.size
|
|
longest = max(w, h)
|
|
if longest > max_side:
|
|
scale = max_side / longest
|
|
img = img.resize((int(w * scale), int(h * scale)), Image.LANCZOS)
|
|
buf = io.BytesIO()
|
|
img.save(buf, format="JPEG", quality=quality, optimize=True)
|
|
return buf.getvalue()
|
|
|
|
|
|
def _image_to_png_bytes(img: Image.Image) -> bytes:
|
|
"""Lossless, full resolution, in memory only (nothing is written to disk)."""
|
|
buf = io.BytesIO()
|
|
img.convert("RGB").save(buf, format="PNG")
|
|
return buf.getvalue()
|
|
|
|
|
|
def _transcribe_image(jpeg_bytes: bytes, model: str) -> str:
|
|
"""One Claude call. The ollama backend lives in ollama.transcribe_png."""
|
|
import anthropic # imported here so --help works without the SDK
|
|
|
|
client = anthropic.Anthropic() # picks up ANTHROPIC_API_KEY
|
|
b64 = base64.standard_b64encode(jpeg_bytes).decode("ascii")
|
|
resp = client.messages.create(
|
|
model=model,
|
|
max_tokens=4096,
|
|
messages=[{
|
|
"role": "user",
|
|
"content": [
|
|
{
|
|
"type": "image",
|
|
"source": {
|
|
"type": "base64",
|
|
"media_type": "image/jpeg",
|
|
"data": b64,
|
|
},
|
|
},
|
|
{"type": "text", "text": TRANSCRIBE_PROMPT},
|
|
],
|
|
}],
|
|
)
|
|
# Concatenate text blocks (usually just one).
|
|
return "".join(b.text for b in resp.content if getattr(b, "type", None) == "text").strip()
|
|
|
|
|
|
def _iter_pages(paths: list[Path], dpi: int = 200) -> list[tuple[str, Callable[[], Image.Image]]]:
|
|
"""Return (label, loader) for every page across all inputs.
|
|
|
|
Pages are rendered only when their loader is called, so a long notebook
|
|
never holds more than one page image in memory.
|
|
"""
|
|
out: list[tuple[str, Callable[[], Image.Image]]] = []
|
|
for p in paths:
|
|
ext = p.suffix.lower()
|
|
if ext == ".pdf":
|
|
import pypdfium2 as pdfium
|
|
|
|
doc = pdfium.PdfDocument(p)
|
|
for i in range(len(doc)):
|
|
# 200 DPI is plenty for handwriting OCR.
|
|
out.append((f"{p.name} p.{i + 1}",
|
|
lambda doc=doc, i=i: doc[i].render(scale=dpi / 72).to_pil()))
|
|
elif ext in SUPPORTED_IMAGE_EXT:
|
|
out.append((p.name, lambda p=p: Image.open(p)))
|
|
else:
|
|
print(f"warning: skipping unsupported file: {p}", file=sys.stderr)
|
|
return out
|
|
|
|
|
|
def _checkpoint_path(first_input: Path) -> Path:
|
|
"""Progress file kept next to the scan while a run is incomplete."""
|
|
return first_input.parent / f".{first_input.stem}.scan-partial.json"
|
|
|
|
|
|
def _checkpoint_key(paths: list[Path], transcribed_by: str, prompt: str, dpi: int) -> dict:
|
|
return {
|
|
"inputs": [[p.name, p.stat().st_size] for p in paths],
|
|
"transcribed_by": transcribed_by,
|
|
"prompt_sha1": hashlib.sha1(prompt.encode()).hexdigest(),
|
|
"dpi": dpi,
|
|
}
|
|
|
|
|
|
def _load_checkpoint(path: Path, key: dict) -> list[dict]:
|
|
"""Pages saved by an earlier, interrupted run with the same settings."""
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, ValueError):
|
|
return []
|
|
if data.get("key") != key:
|
|
print("note: ignoring saved progress from a run with different "
|
|
"inputs, model, prompt, or dpi", file=sys.stderr)
|
|
return []
|
|
return list(data.get("pages", []))
|
|
|
|
|
|
def _save_checkpoint(path: Path, key: dict, done: list[dict]) -> None:
|
|
companion.write_atomic(path, json.dumps({"key": key, "pages": done}))
|
|
|
|
|
|
def to_markdown(results: list[PageResult], title: str) -> str:
|
|
import datetime as _dt
|
|
lines = [
|
|
"---",
|
|
f'title: "{title}"',
|
|
f"extracted: {_dt.date.today().isoformat()}",
|
|
f"pages: {len(results)}",
|
|
"---",
|
|
"",
|
|
f"# {title}",
|
|
"",
|
|
]
|
|
for r in results:
|
|
lines.append(f"## {r.source}")
|
|
lines.append("")
|
|
lines.append(r.markdown if r.markdown else "_(empty)_")
|
|
lines.append("")
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def to_plain_markdown(pages: list[str], title: str, transcribed_by: str) -> str:
|
|
"""Output for ollama runs outside companion mode: <!-- page N --> markers."""
|
|
import datetime as _dt
|
|
return "\n".join([
|
|
"---",
|
|
f'title: "{title}"',
|
|
f"extracted: {_dt.date.today().isoformat()}",
|
|
f"pages: {len(pages)}",
|
|
f"transcribed_by: {transcribed_by}",
|
|
"---",
|
|
"",
|
|
f"# {title}",
|
|
"",
|
|
companion.render_transcript(pages),
|
|
"",
|
|
])
|
|
|
|
|
|
def _fail(msg: str, code: int = 2) -> None:
|
|
print(f"error: {msg}", file=sys.stderr)
|
|
sys.exit(code)
|
|
|
|
|
|
@click.command()
|
|
@click.argument(
|
|
"inputs", nargs=-1, required=True,
|
|
type=click.Path(exists=True, dir_okay=False, path_type=Path),
|
|
)
|
|
@click.option(
|
|
"-o", "--output",
|
|
type=click.Path(dir_okay=False, path_type=Path),
|
|
help="Output .md path. Default: <first input stem>.md",
|
|
)
|
|
@click.option(
|
|
"--backend", type=click.Choice(["claude", "ollama"]), default=None,
|
|
help="Transcription backend. Default: ollama for scans in confidential "
|
|
"folders, else the "
|
|
"config's backend (ollama unless set).",
|
|
)
|
|
@click.option(
|
|
"--host", default=None,
|
|
help="Ollama only: use just this host from the config (e.g. remote, local) "
|
|
"and fail if it is unavailable.",
|
|
)
|
|
@click.option("--model", default=None,
|
|
help=f"Model name. Default: {DEFAULT_MODEL} for claude, the "
|
|
"config's per-host model for ollama.")
|
|
@click.option("--companion/--no-companion", "companion_flag", default=None,
|
|
help="Write the companion <stem>.md next to the scan. Default: on "
|
|
"for scans in confidential folders.")
|
|
@click.option("--allow-cloud", is_flag=True,
|
|
help="Permit --backend claude for scans in confidential folders.")
|
|
@click.option("--config", "config_path",
|
|
type=click.Path(dir_okay=False, path_type=Path), default=None,
|
|
help="Config file. Default: ~/.config/pdf-tools/scan-notes.toml")
|
|
@click.option("--dpi", type=int, default=None, help="PDF render resolution (config default 200).")
|
|
@click.option("--timeout", type=float, default=None,
|
|
help="Ollama per-page timeout in seconds (config default 300).")
|
|
@click.option(
|
|
"--title", default=None,
|
|
help="Title for the markdown document. Default: first input stem.",
|
|
)
|
|
@click.option("--stdout", is_flag=True, help="Write markdown to stdout.")
|
|
def main(inputs: tuple[Path, ...], output: Path | None, backend: str | None,
|
|
host: str | None, model: str | None, companion_flag: bool | None,
|
|
allow_cloud: bool, config_path: Path | None, dpi: int | None,
|
|
timeout: float | None, title: str | None, stdout: bool) -> None:
|
|
"""Transcribe scanned notebook pages to Markdown."""
|
|
cfg = load_config(config_path)
|
|
paths = list(inputs)
|
|
|
|
folders = [cfg.confidential_folder(p) for p in paths]
|
|
confidential = any(folders)
|
|
if confidential and not all(folders):
|
|
_fail("do not mix scans from confidential folders with other files in one run")
|
|
folder = folders[0]
|
|
companion_mode = confidential if companion_flag is None else companion_flag
|
|
if companion_mode and not confidential:
|
|
roots = ", ".join(str(c.root) for c in cfg.confidential) or "none configured"
|
|
_fail(f"--companion needs a scan in a confidential folder ({roots})")
|
|
if companion_mode:
|
|
if len(paths) != 1 or paths[0].suffix.lower() != ".pdf":
|
|
_fail("companion mode takes exactly one PDF")
|
|
if output or stdout:
|
|
_fail("companion mode writes <stem>.md next to the scan; "
|
|
"-o and --stdout need --no-companion")
|
|
|
|
backend = backend or ("ollama" if confidential else cfg.backend)
|
|
if backend not in ("claude", "ollama"):
|
|
_fail(f"unknown backend '{backend}' in config")
|
|
if backend == "claude" and confidential and not allow_cloud:
|
|
_fail(f"scans under the confidential folder {folder.root} are not "
|
|
"sent to Claude unless "
|
|
"--allow-cloud is given; use --backend ollama")
|
|
if host and backend != "ollama":
|
|
_fail("--host applies only to --backend ollama")
|
|
|
|
if backend == "claude":
|
|
if not os.environ.get("ANTHROPIC_API_KEY"):
|
|
_fail("ANTHROPIC_API_KEY not set")
|
|
claude_model = model or DEFAULT_MODEL
|
|
transcribed_by = f"claude/{claude_model}"
|
|
else:
|
|
hosts = cfg.hosts
|
|
if model:
|
|
hosts = [dataclasses.replace(h, model=model) for h in hosts]
|
|
try:
|
|
chosen = select_host(hosts, cfg.probe_timeout_s, forced=host)
|
|
except NoHostAvailable as e:
|
|
_fail(f"{e}. Not falling back to a cloud service.", EXIT_NO_HOST)
|
|
print(f"using Ollama host {chosen.name} "
|
|
f"({chosen.url}, {chosen.model})", file=sys.stderr)
|
|
transcribed_by = provenance(chosen)
|
|
# Global names and prompt_extra text, plus those of every folder
|
|
# the inputs come from.
|
|
names = dict.fromkeys(n for f in folders for n in cfg.names_for(f))
|
|
extras = dict.fromkeys(e for f in folders for e in cfg.extras_for(f))
|
|
prompt = build_prompt(list(names), list(extras))
|
|
page_timeout = timeout or cfg.timeout_s
|
|
|
|
dpi = dpi or cfg.dpi
|
|
pages = _iter_pages(paths, dpi=dpi)
|
|
if not pages:
|
|
_fail("no usable pages found", 1)
|
|
|
|
# Progress is saved after every page, so an interrupted run resumes
|
|
# where it stopped. The file sits next to the scan and is removed once
|
|
# the output is written.
|
|
ckpt = _checkpoint_path(paths[0])
|
|
key = _checkpoint_key(paths, transcribed_by,
|
|
TRANSCRIBE_PROMPT if backend == "claude" else prompt, dpi)
|
|
done = _load_checkpoint(ckpt, key)[:len(pages)]
|
|
if done:
|
|
print(f"resuming after page {len(done)} of {len(pages)}", file=sys.stderr)
|
|
|
|
stats_log = StatsLog(cfg.stats_log if backend == "ollama" else None)
|
|
run_id = f"{time.strftime('%Y%m%dT%H%M%S')}-{os.getpid()}"
|
|
loaded: dict | None = None # /api/ps snapshot, taken once the model is loaded
|
|
|
|
for i, (label, load) in enumerate(pages, start=1):
|
|
if i <= len(done):
|
|
continue
|
|
if backend == "claude":
|
|
print(f"[{i}/{len(pages)}] {label} ...", file=sys.stderr)
|
|
jpeg = _image_to_jpeg_bytes(load())
|
|
md = _transcribe_image(jpeg, model=claude_model)
|
|
else:
|
|
print(f"[{i}/{len(pages)}] {label} on {chosen.name} ...",
|
|
end="", file=sys.stderr, flush=True)
|
|
t0 = time.monotonic()
|
|
img = load()
|
|
try:
|
|
page = transcribe_png(_image_to_png_bytes(img), chosen, prompt,
|
|
page_timeout, cfg.max_output_tokens)
|
|
except RepetitionError:
|
|
print("", file=sys.stderr)
|
|
print(f"warning: page {i} kept looping after {len(REPEAT_RETRIES)} "
|
|
"retries; writing a placeholder", file=sys.stderr)
|
|
page = PageOutput(REPEAT_PLACEHOLDER, {
|
|
"prompt_tokens": 0, "output_tokens": 0, "context_tokens": 0,
|
|
"done_reason": "repeat_abort", "retries": len(REPEAT_RETRIES),
|
|
"total_s": round(time.monotonic() - t0, 2)})
|
|
except OllamaError as e:
|
|
print("", file=sys.stderr)
|
|
saved = (f"pages 1-{len(done)} are saved; rerun to resume"
|
|
if done else "nothing was saved")
|
|
_fail(f"page {i}: {e}. {saved}.", EXIT_TRANSCRIBE_FAILED)
|
|
md, st = page.text, page.stats
|
|
if loaded is None:
|
|
loaded = loaded_model(chosen, cfg.probe_timeout_s)
|
|
retried = f", {st['retries']} retries" if st.get("retries") else ""
|
|
print(f" {time.monotonic() - t0:.0f} s, {st['context_tokens']} tokens{retried}",
|
|
file=sys.stderr)
|
|
nctx = loaded.get("loaded_num_ctx") or chosen.num_ctx
|
|
if st["done_reason"] == "length":
|
|
print(f"warning: page {i} output stopped at {st['output_tokens']} "
|
|
"tokens after retries; marked as cut off", file=sys.stderr)
|
|
elif nctx and st["context_tokens"] >= 0.9 * nctx:
|
|
print(f"warning: page {i} used {st['context_tokens']} of "
|
|
f"{nctx} context tokens", file=sys.stderr)
|
|
stats_log.write({
|
|
"time": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
|
"run": run_id,
|
|
"host": urlparse(chosen.url).hostname,
|
|
"model": chosen.model,
|
|
"num_ctx_setting": chosen.num_ctx,
|
|
"dpi": dpi,
|
|
"label": label,
|
|
"page": i,
|
|
"image_px": list(img.size),
|
|
**st,
|
|
**loaded,
|
|
})
|
|
done.append({"label": label, "text": md})
|
|
_save_checkpoint(ckpt, key, done)
|
|
|
|
texts = [d["text"] for d in done]
|
|
results = [PageResult(index=i, source=d["label"], markdown=d["text"])
|
|
for i, d in enumerate(done, start=1)]
|
|
|
|
if companion_mode:
|
|
path, created, warnings = companion.write_companion(
|
|
paths[0], folder.root, texts, transcribed_by, folder.companion,
|
|
folder.titles)
|
|
for w in warnings:
|
|
print(f"warning: {w}", file=sys.stderr)
|
|
verb = "created" if created else "updated draft transcript in"
|
|
print(f"{verb} {path}", file=sys.stderr)
|
|
ckpt.unlink(missing_ok=True)
|
|
return
|
|
|
|
doc_title = title or paths[0].stem
|
|
if backend == "claude":
|
|
md_out = to_markdown(results, doc_title)
|
|
else:
|
|
md_out = to_plain_markdown(texts, doc_title, transcribed_by)
|
|
|
|
if stdout:
|
|
sys.stdout.write(md_out)
|
|
ckpt.unlink(missing_ok=True)
|
|
return
|
|
|
|
out_path = output or paths[0].with_suffix(".md")
|
|
companion.write_atomic(out_path, md_out)
|
|
ckpt.unlink(missing_ok=True)
|
|
print(f"wrote {len(results)} pages -> {out_path}", file=sys.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|