diff --git a/build_store.py b/build_store.py index fd2b477..551545c 100644 --- a/build_store.py +++ b/build_store.py @@ -125,8 +125,16 @@ def get_embed_tokenizer(): global _EMBED_TOKENIZER if _EMBED_TOKENIZER is None: from transformers import AutoTokenizer + # model_max_length is raised because this tokenizer is used ONLY to + # count tokens for chunking -- the model never sees these sequences. + # Left at its default (512) it warns on every document longer than + # that ("Token indices sequence length is longer than..."), which is + # noise here, not a problem. Counts are unaffected either way. _EMBED_TOKENIZER = AutoTokenizer.from_pretrained( - EMBED_MODEL_NAME, cache_dir="./models", local_files_only=True + EMBED_MODEL_NAME, + cache_dir="./models", + local_files_only=True, + model_max_length=int(1e9), ) return _EMBED_TOKENIZER diff --git a/clippings_search/build_clippings.py b/clippings_search/build_clippings.py index 90de713..1d1b369 100644 --- a/clippings_search/build_clippings.py +++ b/clippings_search/build_clippings.py @@ -73,8 +73,16 @@ def get_embed_tokenizer(): global _EMBED_TOKENIZER if _EMBED_TOKENIZER is None: from transformers import AutoTokenizer + # model_max_length is raised because this tokenizer is used ONLY to + # count tokens for chunking -- the model never sees these sequences. + # Left at its default (512) it warns on every document longer than + # that ("Token indices sequence length is longer than..."), which is + # noise here, not a problem. Counts are unaffected either way. _EMBED_TOKENIZER = AutoTokenizer.from_pretrained( - EMBED_MODEL_NAME, cache_dir="./models", local_files_only=True + EMBED_MODEL_NAME, + cache_dir="./models", + local_files_only=True, + model_max_length=int(1e9), ) return _EMBED_TOKENIZER