unify tiktoken model

crazy_functions/理解PDF文档内容.py

@@ -18,7 +18,7 @@ def 解析PDF(file_name, llm_kwargs, plugin_kwargs, chatbot, history, system_pro
 
     from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf
     from toolbox import get_conf
-    enc = tiktoken.encoding_for_model(*get_conf('LLM_MODEL'))
+    enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
     def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
     paper_fragments = breakdown_txt_to_satisfy_token_limit_for_pdf(
         txt=file_content,  get_token_fn=get_token_num, limit=TOKEN_LIMIT_PER_FRAGMENT)