diff --git a/README.md b/README.md index 18ca814..17402e2 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ ssearch/ │ ├── retrieve_clippings.py # Verbatim clippings chunk retrieval │ └── store_clippings/ # Persisted clippings vector store (ChromaDB) ├── data/ # Symlink to journal .txt files -├── clippings/ # Symlink to clippings (PDFs, TXT, webarchive, RTF) +├── clippings/ # Symlink to clippings (PDFs, TXT, Markdown, webarchive, RTF) ├── store/ # Persisted journal vector store ├── models/ # Cached HuggingFace models (offline) ├── pyproject.toml # Project metadata and dependencies (uv) @@ -118,7 +118,7 @@ uv run python clippings_search/build_clippings.py --rebuild The default incremental mode loads the existing index, compares file sizes and modification dates, and only re-indexes what changed. A full rebuild is only needed when chunk parameters, the embedding model, or the metadata policy change. -`build_clippings.py` handles PDFs, TXT, webarchive, and RTF files. PDFs are validated before indexing — those without extractable text are skipped and written to `ocr_needed.txt` for later OCR. +`build_clippings.py` handles PDFs, TXT, Markdown, webarchive, and RTF files. PDFs are validated before indexing — those without extractable text are skipped and written to `ocr_needed.txt` for later OCR. ### Search journals diff --git a/clippings_search/build_clippings.py b/clippings_search/build_clippings.py index 1d1b369..fb66d89 100644 --- a/clippings_search/build_clippings.py +++ b/clippings_search/build_clippings.py @@ -5,7 +5,7 @@ # Default mode (incremental): loads the existing index and adds only # new or modified files. Use --rebuild for a full rebuild from scratch. # -# Handles PDFs, TXT, webarchive, and RTF files. Skips non-extractable PDFs +# Handles PDFs, TXT, Markdown, webarchive, and RTF files. Skips non-extractable PDFs # and writes them to ocr_needed.txt for later OCR processing. # # February 2026 @@ -50,7 +50,9 @@ CHUNK_SIZE = 256 CHUNK_OVERLAP = 25 # File types handled by SimpleDirectoryReader (PDF + TXT) -READER_EXTS = {".pdf", ".txt"} +# .md is read as plain text by SimpleDirectoryReader (one document, full +# text, standard metadata) -- no special Markdown handling needed. +READER_EXTS = {".pdf", ".txt", ".md"} # File types handled by custom loaders CUSTOM_EXTS = {".webarchive", ".rtf"} # All supported extensions @@ -241,7 +243,13 @@ def scan_clippings(): continue reader_files.append(fpath) - elif ext == ".txt": + elif ext in READER_EXTS: + # Everything else in READER_EXTS is plain text that + # SimpleDirectoryReader handles directly (.txt, .md). Keyed off the + # set rather than a literal so adding an extension there is enough; + # previously this was `elif ext == ".txt"`, so .md passed the + # SUPPORTED_EXTS guard and then fell through every branch below, + # silently indexing nothing and not even counting as skipped. reader_files.append(fpath) elif ext == ".webarchive": @@ -258,6 +266,11 @@ def scan_clippings(): else: skipped.append((fpath, "no extractable text from RTF")) + else: + # Reachable only if SUPPORTED_EXTS gains a type with no handler. + # Make that loud rather than silently dropping the file. + skipped.append((fpath, f"NO HANDLER for supported type {ext}")) + return reader_files, custom_docs, skipped, ocr_needed