镜像自地址
https://github.com/binary-husky/gpt_academic.git
已同步 2025-12-06 06:26:47 +00:00
unify tiktoken model
这个提交包含在:
@@ -13,7 +13,7 @@ class PaperFileGroup():
|
||||
# count_token
|
||||
import tiktoken
|
||||
from toolbox import get_conf
|
||||
enc = tiktoken.encoding_for_model(*get_conf('LLM_MODEL'))
|
||||
enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
|
||||
def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
|
||||
self.get_token_num = get_token_num
|
||||
|
||||
|
||||
在新工单中引用
屏蔽一个用户