pdf-tools/src/pdf_tools/extract_scan.py
Eric Furst e464871b69 Initial public release of pdf-tools
Command-line tools for marked-up PDFs and scanned handwritten notes:
pdf-scrub, pdf-notes-md, scan-notes-md (Ollama by default, Claude on
request), scan-notes-index (per-page search metadata), and
scan-notes-stats.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011ZSvbNaLhgFuTm7wmkgbWs
2026-11-03 07:25:51 -05:00

439 lines
17 KiB
Python

"""Tool 3: extract notes from scanned notebook pages to Markdown.
Accepts either:
- One or more image files (jpg/png/heic/tiff)
- A PDF (each page is rasterized and processed)
Two backends:
- ollama (default): a vision model on a machine you own, through the Ollama HTTP
API (see ollama.py). Hosts are tried in the order given in the config
file (see scan_config.py). There is no fallback to any cloud service.
- claude: the Anthropic API, used only when asked for with --backend
claude or the config. Set ANTHROPIC_API_KEY. Model defaults to Claude
Opus; override with --model.
Scans under a confidential folder (declared in the config file; none are
built in) default to the ollama backend, refuse the claude backend unless
--allow-cloud is given, and are written to a companion Markdown file shaped
by that folder's settings (see companion.py).
"""
from __future__ import annotations
import base64
import dataclasses
import hashlib
import io
import json
import os
import sys
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Callable
from urllib.parse import urlparse
import click
from PIL import Image
from . import companion
from .ollama import (REPEAT_PLACEHOLDER, REPEAT_RETRIES, NoHostAvailable,
OllamaError, PageOutput, RepetitionError, build_prompt,
loaded_model, provenance, select_host, transcribe_png)
from .scan_config import load_config
from .scan_stats import StatsLog
# Anthropic is imported lazily inside _transcribe_image so the rest of
# the module is usable without the API key set (e.g. for --help), and so
# the ollama backend never loads it.
# Reasonable defaults. The model is one of the few things you might
# want to tweak per-run as new versions ship. Opus is the default
# because handwriting transcription is the one place accuracy
# clearly justifies the cost.
DEFAULT_MODEL = "claude-opus-5-5"
SUPPORTED_IMAGE_EXT = {".jpg", ".jpeg", ".png", ".webp", ".gif", ".tif", ".tiff", ".heic"}
# Exit codes beyond click's usual 1 and 2.
EXIT_NO_HOST = 3
EXIT_TRANSCRIBE_FAILED = 4
TRANSCRIBE_PROMPT = """\
You are transcribing a scanned page from a handwritten research notebook.
Output requirements:
- Reproduce the page content as clean Markdown.
- Preserve structure: headings, bullet points, indentation, numbered lists.
- Render equations and chemical formulas in LaTeX. Inline math uses $...$, display math uses $$...$$.
- Preserve units, subscripts, and Greek letters faithfully.
- For figures, diagrams, or sketches, insert a single line like:
`[FIGURE: brief description of what is drawn]`
- For text you cannot read with confidence, use `[?]` for a single illegible word
or `[??: best guess]` if you have a tentative reading.
- Do NOT add commentary, summaries, or explanations. Transcribe only.
- Do NOT wrap your answer in code fences. Output raw Markdown directly.
If the page is blank or contains no notebook content, output exactly:
[BLANK PAGE]
"""
@dataclass
class PageResult:
index: int # 1-indexed page/image number
source: str # filename or "page N of source.pdf"
markdown: str
def _image_to_jpeg_bytes(img: Image.Image, max_side: int = 2000, quality: int = 85) -> bytes:
"""Downscale + re-encode to keep API payloads reasonable."""
img = img.convert("RGB")
w, h = img.size
longest = max(w, h)
if longest > max_side:
scale = max_side / longest
img = img.resize((int(w * scale), int(h * scale)), Image.LANCZOS)
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=quality, optimize=True)
return buf.getvalue()
def _image_to_png_bytes(img: Image.Image) -> bytes:
"""Lossless, full resolution, in memory only (nothing is written to disk)."""
buf = io.BytesIO()
img.convert("RGB").save(buf, format="PNG")
return buf.getvalue()
def _transcribe_image(jpeg_bytes: bytes, model: str) -> str:
"""One Claude call. The ollama backend lives in ollama.transcribe_png."""
import anthropic # imported here so --help works without the SDK
client = anthropic.Anthropic() # picks up ANTHROPIC_API_KEY
b64 = base64.standard_b64encode(jpeg_bytes).decode("ascii")
resp = client.messages.create(
model=model,
max_tokens=4096,
messages=[{
"role": "user",
"content": [
{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/jpeg",
"data": b64,
},
},
{"type": "text", "text": TRANSCRIBE_PROMPT},
],
}],
)
# Concatenate text blocks (usually just one).
return "".join(b.text for b in resp.content if getattr(b, "type", None) == "text").strip()
def _iter_pages(paths: list[Path], dpi: int = 200) -> list[tuple[str, Callable[[], Image.Image]]]:
"""Return (label, loader) for every page across all inputs.
Pages are rendered only when their loader is called, so a long notebook
never holds more than one page image in memory.
"""
out: list[tuple[str, Callable[[], Image.Image]]] = []
for p in paths:
ext = p.suffix.lower()
if ext == ".pdf":
import pypdfium2 as pdfium
doc = pdfium.PdfDocument(p)
for i in range(len(doc)):
# 200 DPI is plenty for handwriting OCR.
out.append((f"{p.name} p.{i + 1}",
lambda doc=doc, i=i: doc[i].render(scale=dpi / 72).to_pil()))
elif ext in SUPPORTED_IMAGE_EXT:
out.append((p.name, lambda p=p: Image.open(p)))
else:
print(f"warning: skipping unsupported file: {p}", file=sys.stderr)
return out
def _checkpoint_path(first_input: Path) -> Path:
"""Progress file kept next to the scan while a run is incomplete."""
return first_input.parent / f".{first_input.stem}.scan-partial.json"
def _checkpoint_key(paths: list[Path], transcribed_by: str, prompt: str, dpi: int) -> dict:
return {
"inputs": [[p.name, p.stat().st_size] for p in paths],
"transcribed_by": transcribed_by,
"prompt_sha1": hashlib.sha1(prompt.encode()).hexdigest(),
"dpi": dpi,
}
def _load_checkpoint(path: Path, key: dict) -> list[dict]:
"""Pages saved by an earlier, interrupted run with the same settings."""
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return []
if data.get("key") != key:
print("note: ignoring saved progress from a run with different "
"inputs, model, prompt, or dpi", file=sys.stderr)
return []
return list(data.get("pages", []))
def _save_checkpoint(path: Path, key: dict, done: list[dict]) -> None:
companion.write_atomic(path, json.dumps({"key": key, "pages": done}))
def to_markdown(results: list[PageResult], title: str) -> str:
import datetime as _dt
lines = [
"---",
f'title: "{title}"',
f"extracted: {_dt.date.today().isoformat()}",
f"pages: {len(results)}",
"---",
"",
f"# {title}",
"",
]
for r in results:
lines.append(f"## {r.source}")
lines.append("")
lines.append(r.markdown if r.markdown else "_(empty)_")
lines.append("")
return "\n".join(lines) + "\n"
def to_plain_markdown(pages: list[str], title: str, transcribed_by: str) -> str:
"""Output for ollama runs outside companion mode: <!-- page N --> markers."""
import datetime as _dt
return "\n".join([
"---",
f'title: "{title}"',
f"extracted: {_dt.date.today().isoformat()}",
f"pages: {len(pages)}",
f"transcribed_by: {transcribed_by}",
"---",
"",
f"# {title}",
"",
companion.render_transcript(pages),
"",
])
def _fail(msg: str, code: int = 2) -> None:
print(f"error: {msg}", file=sys.stderr)
sys.exit(code)
@click.command()
@click.argument(
"inputs", nargs=-1, required=True,
type=click.Path(exists=True, dir_okay=False, path_type=Path),
)
@click.option(
"-o", "--output",
type=click.Path(dir_okay=False, path_type=Path),
help="Output .md path. Default: <first input stem>.md",
)
@click.option(
"--backend", type=click.Choice(["claude", "ollama"]), default=None,
help="Transcription backend. Default: ollama for scans in confidential "
"folders, else the "
"config's backend (ollama unless set).",
)
@click.option(
"--host", default=None,
help="Ollama only: use just this host from the config (e.g. remote, local) "
"and fail if it is unavailable.",
)
@click.option("--model", default=None,
help=f"Model name. Default: {DEFAULT_MODEL} for claude, the "
"config's per-host model for ollama.")
@click.option("--companion/--no-companion", "companion_flag", default=None,
help="Write the companion <stem>.md next to the scan. Default: on "
"for scans in confidential folders.")
@click.option("--allow-cloud", is_flag=True,
help="Permit --backend claude for scans in confidential folders.")
@click.option("--config", "config_path",
type=click.Path(dir_okay=False, path_type=Path), default=None,
help="Config file. Default: ~/.config/pdf-tools/scan-notes.toml")
@click.option("--dpi", type=int, default=None, help="PDF render resolution (config default 200).")
@click.option("--timeout", type=float, default=None,
help="Ollama per-page timeout in seconds (config default 300).")
@click.option(
"--title", default=None,
help="Title for the markdown document. Default: first input stem.",
)
@click.option("--stdout", is_flag=True, help="Write markdown to stdout.")
def main(inputs: tuple[Path, ...], output: Path | None, backend: str | None,
host: str | None, model: str | None, companion_flag: bool | None,
allow_cloud: bool, config_path: Path | None, dpi: int | None,
timeout: float | None, title: str | None, stdout: bool) -> None:
"""Transcribe scanned notebook pages to Markdown."""
cfg = load_config(config_path)
paths = list(inputs)
folders = [cfg.confidential_folder(p) for p in paths]
confidential = any(folders)
if confidential and not all(folders):
_fail("do not mix scans from confidential folders with other files in one run")
folder = folders[0]
companion_mode = confidential if companion_flag is None else companion_flag
if companion_mode and not confidential:
roots = ", ".join(str(c.root) for c in cfg.confidential) or "none configured"
_fail(f"--companion needs a scan in a confidential folder ({roots})")
if companion_mode:
if len(paths) != 1 or paths[0].suffix.lower() != ".pdf":
_fail("companion mode takes exactly one PDF")
if output or stdout:
_fail("companion mode writes <stem>.md next to the scan; "
"-o and --stdout need --no-companion")
backend = backend or ("ollama" if confidential else cfg.backend)
if backend not in ("claude", "ollama"):
_fail(f"unknown backend '{backend}' in config")
if backend == "claude" and confidential and not allow_cloud:
_fail(f"scans under the confidential folder {folder.root} are not "
"sent to Claude unless "
"--allow-cloud is given; use --backend ollama")
if host and backend != "ollama":
_fail("--host applies only to --backend ollama")
if backend == "claude":
if not os.environ.get("ANTHROPIC_API_KEY"):
_fail("ANTHROPIC_API_KEY not set")
claude_model = model or DEFAULT_MODEL
transcribed_by = f"claude/{claude_model}"
else:
hosts = cfg.hosts
if model:
hosts = [dataclasses.replace(h, model=model) for h in hosts]
try:
chosen = select_host(hosts, cfg.probe_timeout_s, forced=host)
except NoHostAvailable as e:
_fail(f"{e}. Not falling back to a cloud service.", EXIT_NO_HOST)
print(f"using Ollama host {chosen.name} "
f"({chosen.url}, {chosen.model})", file=sys.stderr)
transcribed_by = provenance(chosen)
# Global names and prompt_extra text, plus those of every folder
# the inputs come from.
names = dict.fromkeys(n for f in folders for n in cfg.names_for(f))
extras = dict.fromkeys(e for f in folders for e in cfg.extras_for(f))
prompt = build_prompt(list(names), list(extras))
page_timeout = timeout or cfg.timeout_s
dpi = dpi or cfg.dpi
pages = _iter_pages(paths, dpi=dpi)
if not pages:
_fail("no usable pages found", 1)
# Progress is saved after every page, so an interrupted run resumes
# where it stopped. The file sits next to the scan and is removed once
# the output is written.
ckpt = _checkpoint_path(paths[0])
key = _checkpoint_key(paths, transcribed_by,
TRANSCRIBE_PROMPT if backend == "claude" else prompt, dpi)
done = _load_checkpoint(ckpt, key)[:len(pages)]
if done:
print(f"resuming after page {len(done)} of {len(pages)}", file=sys.stderr)
stats_log = StatsLog(cfg.stats_log if backend == "ollama" else None)
run_id = f"{time.strftime('%Y%m%dT%H%M%S')}-{os.getpid()}"
loaded: dict | None = None # /api/ps snapshot, taken once the model is loaded
for i, (label, load) in enumerate(pages, start=1):
if i <= len(done):
continue
if backend == "claude":
print(f"[{i}/{len(pages)}] {label} ...", file=sys.stderr)
jpeg = _image_to_jpeg_bytes(load())
md = _transcribe_image(jpeg, model=claude_model)
else:
print(f"[{i}/{len(pages)}] {label} on {chosen.name} ...",
end="", file=sys.stderr, flush=True)
t0 = time.monotonic()
img = load()
try:
page = transcribe_png(_image_to_png_bytes(img), chosen, prompt,
page_timeout, cfg.max_output_tokens)
except RepetitionError:
print("", file=sys.stderr)
print(f"warning: page {i} kept looping after {len(REPEAT_RETRIES)} "
"retries; writing a placeholder", file=sys.stderr)
page = PageOutput(REPEAT_PLACEHOLDER, {
"prompt_tokens": 0, "output_tokens": 0, "context_tokens": 0,
"done_reason": "repeat_abort", "retries": len(REPEAT_RETRIES),
"total_s": round(time.monotonic() - t0, 2)})
except OllamaError as e:
print("", file=sys.stderr)
saved = (f"pages 1-{len(done)} are saved; rerun to resume"
if done else "nothing was saved")
_fail(f"page {i}: {e}. {saved}.", EXIT_TRANSCRIBE_FAILED)
md, st = page.text, page.stats
if loaded is None:
loaded = loaded_model(chosen, cfg.probe_timeout_s)
retried = f", {st['retries']} retries" if st.get("retries") else ""
print(f" {time.monotonic() - t0:.0f} s, {st['context_tokens']} tokens{retried}",
file=sys.stderr)
nctx = loaded.get("loaded_num_ctx") or chosen.num_ctx
if st["done_reason"] == "length":
print(f"warning: page {i} output stopped at {st['output_tokens']} "
"tokens after retries; marked as cut off", file=sys.stderr)
elif nctx and st["context_tokens"] >= 0.9 * nctx:
print(f"warning: page {i} used {st['context_tokens']} of "
f"{nctx} context tokens", file=sys.stderr)
stats_log.write({
"time": time.strftime("%Y-%m-%dT%H:%M:%S"),
"run": run_id,
"host": urlparse(chosen.url).hostname,
"model": chosen.model,
"num_ctx_setting": chosen.num_ctx,
"dpi": dpi,
"label": label,
"page": i,
"image_px": list(img.size),
**st,
**loaded,
})
done.append({"label": label, "text": md})
_save_checkpoint(ckpt, key, done)
texts = [d["text"] for d in done]
results = [PageResult(index=i, source=d["label"], markdown=d["text"])
for i, d in enumerate(done, start=1)]
if companion_mode:
path, created, warnings = companion.write_companion(
paths[0], folder.root, texts, transcribed_by, folder.companion,
folder.titles)
for w in warnings:
print(f"warning: {w}", file=sys.stderr)
verb = "created" if created else "updated draft transcript in"
print(f"{verb} {path}", file=sys.stderr)
ckpt.unlink(missing_ok=True)
return
doc_title = title or paths[0].stem
if backend == "claude":
md_out = to_markdown(results, doc_title)
else:
md_out = to_plain_markdown(texts, doc_title, transcribed_by)
if stdout:
sys.stdout.write(md_out)
ckpt.unlink(missing_ok=True)
return
out_path = output or paths[0].with_suffix(".md")
companion.write_atomic(out_path, md_out)
ckpt.unlink(missing_ok=True)
print(f"wrote {len(results)} pages -> {out_path}", file=sys.stderr)
if __name__ == "__main__":
main()