furst/ssearch
0
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Silence tokenizer length warning

Browse source
This commit is contained in:
Eric Furst 2026-09-20 13:00:00 -04:00
commit 03c31641b7
2 changed files with 18 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

10
clippings_search/build_clippings.py
View file

@ -73,8 +73,16 @@ def get_embed_tokenizer():
global _EMBED_TOKENIZER
if _EMBED_TOKENIZER is None:
from transformers import AutoTokenizer
# model_max_length is raised because this tokenizer is used ONLY to
# count tokens for chunking -- the model never sees these sequences.
# Left at its default (512) it warns on every document longer than
# that ("Token indices sequence length is longer than..."), which is
# noise here, not a problem. Counts are unaffected either way.
_EMBED_TOKENIZER = AutoTokenizer.from_pretrained(
EMBED_MODEL_NAME, cache_dir="./models", local_files_only=True
EMBED_MODEL_NAME,
cache_dir="./models",
local_files_only=True,
model_max_length=int(1e9),
)
return _EMBED_TOKENIZER