int/gpt_academic
1
0
派生 0
镜像自地址 https://github.com/binary-husky/gpt_academic.git 已同步 2025-12-06 06:26:47 +00:00
代码 工单 软件包 项目 版本发布 百科 动态

Merge Latest Frontier (#1991)

浏览代码 
* logging sys to loguru: stage 1 complete

* import loguru: stage 2

* logging -> loguru: stage 3

* support o1-preview and o1-mini

* logging -> loguru stage 4

* update social helper

* logging -> loguru: final stage

* fix: console output

* update translation matrix

* fix: loguru argument error with proxy enabled (#1977)

* relax llama index version

* remove comment

* Added some modules to support openrouter (#1975)

* Added some modules for supporting openrouter model

Added some modules for supporting openrouter model

* Update config.py

* Update .gitignore

* Update bridge_openrouter.py

* Not changed actually

* Refactor logging in bridge_openrouter.py

---------

Co-authored-by: binary-husky <qingxu.fu@outlook.com>

* remove logging extra

---------

Co-authored-by: Steven Moder <java20131114@gmail.com>
Co-authored-by: Ren Lifei <2602264455@qq.com>
这个提交包含在：
binary-husky
2024-10-05 17:09:18 +08:00
提交者 GitHub
父节点 597c320808
当前提交 a01ca93362
共有 91 个文件被更改，包括 2558 次插入 和 742 次删除
下载 Patch 文件 下载 Diff 文件 展开所有文件 折叠所有文件

12
shared_utils/advanced_markdown_format.py
查看文件

@@ -2,6 +2,8 @@ import markdown
import re
import os
import math
from loguru import logger
from textwrap import dedent
from functools import lru_cache
from pymdownx.superfences import fence_code_format
@@ -227,14 +229,14 @@ def fix_dollar_sticking_bug(txt):
if single_stack_height > 0:
if txt[:(index+1)].find('\n') > 0 or txt[:(index+1)].find('<td>') > 0 or txt[:(index+1)].find('</td>') > 0:
print('公式之中出现了异常 (Unexpect element in equation)')
logger.error('公式之中出现了异常 (Unexpect element in equation)')
single_stack_height = 0
txt_result += ' $'
continue
if double_stack_height > 0:
if txt[:(index+1)].find('\n\n') > 0:
print('公式之中出现了异常 (Unexpect element in equation)')
logger.error('公式之中出现了异常 (Unexpect element in equation)')
double_stack_height = 0
txt_result += '$$'
continue
@@ -253,13 +255,13 @@ def fix_dollar_sticking_bug(txt):
txt = txt[(index+2):]
else:
if double_stack_height != 0:
# print(txt[:(index)])
print('发现异常嵌套公式')
# logger.info(txt[:(index)])
logger.info('发现异常嵌套公式')
if single_stack_height == 0:
single_stack_height = 1
else:
single_stack_height = 0
# print(txt[:(index)])
# logger.info(txt[:(index)])
txt_result += txt[:(index+1)]
txt = txt[(index+1):]
break

27
shared_utils/colorful.py
查看文件

@@ -1,5 +1,6 @@
import platform
from sys import stdout
from loguru import logger
if platform.system()=="Linux":
pass
@@ -59,3 +60,29 @@ def sprint亮紫(*kw):
return "\033[1;35m"+' '.join(kw)+"\033[0m"
def sprint亮靛(*kw):
return "\033[1;36m"+' '.join(kw)+"\033[0m"
def log红(*kw,**kargs):
logger.opt(depth=1).info(sprint红(*kw))
def log绿(*kw,**kargs):
logger.opt(depth=1).info(sprint绿(*kw))
def log黄(*kw,**kargs):
logger.opt(depth=1).info(sprint黄(*kw))
def log蓝(*kw,**kargs):
logger.opt(depth=1).info(sprint蓝(*kw))
def log紫(*kw,**kargs):
logger.opt(depth=1).info(sprint紫(*kw))
def log靛(*kw,**kargs):
logger.opt(depth=1).info(sprint靛(*kw))
def log亮红(*kw,**kargs):
logger.opt(depth=1).info(sprint亮红(*kw))
def log亮绿(*kw,**kargs):
logger.opt(depth=1).info(sprint亮绿(*kw))
def log亮黄(*kw,**kargs):
logger.opt(depth=1).info(sprint亮黄(*kw))
def log亮蓝(*kw,**kargs):
logger.opt(depth=1).info(sprint亮蓝(*kw))
def log亮紫(*kw,**kargs):
logger.opt(depth=1).info(sprint亮紫(*kw))
def log亮靛(*kw,**kargs):
logger.opt(depth=1).info(sprint亮靛(*kw))

26
shared_utils/config_loader.py
查看文件

@@ -2,7 +2,7 @@ import importlib
import time
import os
from functools import lru_cache
from shared_utils.colorful import print亮红, print亮绿, print亮蓝
from shared_utils.colorful import log亮红, log亮绿, log亮蓝
pj = os.path.join
default_user_name = 'default_user'
@@ -30,13 +30,13 @@ def read_env_variable(arg, default_value):
env_arg = os.environ[arg]
else:
raise KeyError
print(f"[ENV_VAR] 尝试加载{arg}，默认值：{default_value} --> 修正值：{env_arg}")
log亮绿(f"[ENV_VAR] 尝试加载{arg}，默认值：{default_value} --> 修正值：{env_arg}")
try:
if isinstance(default_value, bool):
env_arg = env_arg.strip()
if env_arg == 'True': r = True
elif env_arg == 'False': r = False
else: print('Enter True or False, but have:', env_arg); r = default_value
else: log亮红('Expect `True` or `False`, but have:', env_arg); r = default_value
elif isinstance(default_value, int):
r = int(env_arg)
elif isinstance(default_value, float):
@@ -51,13 +51,13 @@ def read_env_variable(arg, default_value):
assert arg == "proxies"
r = eval(env_arg)
else:
print亮红(f"[ENV_VAR] 环境变量{arg}不支持通过环境变量设置! ")
log亮红(f"[ENV_VAR] 环境变量{arg}不支持通过环境变量设置! ")
raise KeyError
except:
print亮红(f"[ENV_VAR] 环境变量{arg}加载失败! ")
log亮红(f"[ENV_VAR] 环境变量{arg}加载失败! ")
raise KeyError(f"[ENV_VAR] 环境变量{arg}加载失败! ")
print亮绿(f"[ENV_VAR] 成功读取环境变量{arg}")
log亮绿(f"[ENV_VAR] 成功读取环境变量{arg}")
return r
@@ -80,21 +80,21 @@ def read_single_conf_with_lru_cache(arg):
if arg == 'API_URL_REDIRECT':
oai_rd = r.get("https://api.openai.com/v1/chat/completions", None) # API_URL_REDIRECT填写格式是错误的，请阅读`https://github.com/binary-husky/gpt_academic/wiki/项目配置说明`
if oai_rd and not oai_rd.endswith('/completions'):
print亮红("\n\n[API_URL_REDIRECT] API_URL_REDIRECT填错了。请阅读`https://github.com/binary-husky/gpt_academic/wiki/项目配置说明`。如果您确信自己没填错，无视此消息即可。")
log亮红("\n\n[API_URL_REDIRECT] API_URL_REDIRECT填错了。请阅读`https://github.com/binary-husky/gpt_academic/wiki/项目配置说明`。如果您确信自己没填错，无视此消息即可。")
time.sleep(5)
if arg == 'API_KEY':
print亮蓝(f"[API_KEY] 本项目现已支持OpenAI和Azure的api-key。也支持同时填写多个api-key，如API_KEY=\"openai-key1,openai-key2,azure-key3\"")
print亮蓝(f"[API_KEY] 您既可以在config.py中修改api-key(s)，也可以在问题输入区输入临时的api-key(s)，然后回车键提交后即可生效。")
log亮蓝(f"[API_KEY] 本项目现已支持OpenAI和Azure的api-key。也支持同时填写多个api-key，如API_KEY=\"openai-key1,openai-key2,azure-key3\"")
log亮蓝(f"[API_KEY] 您既可以在config.py中修改api-key(s)，也可以在问题输入区输入临时的api-key(s)，然后回车键提交后即可生效。")
if is_any_api_key(r):
print亮绿(f"[API_KEY] 您的 API_KEY 是: {r[:15]}*** API_KEY 导入成功")
log亮绿(f"[API_KEY] 您的 API_KEY 是: {r[:15]}*** API_KEY 导入成功")
else:
print亮红(f"[API_KEY] 您的 API_KEY（{r[:15]}***）不满足任何一种已知的密钥格式，请在config文件中修改API密钥之后再运行（详见`https://github.com/binary-husky/gpt_academic/wiki/api_key`）。")
log亮红(f"[API_KEY] 您的 API_KEY（{r[:15]}***）不满足任何一种已知的密钥格式，请在config文件中修改API密钥之后再运行（详见`https://github.com/binary-husky/gpt_academic/wiki/api_key`）。")
if arg == 'proxies':
if not read_single_conf_with_lru_cache('USE_PROXY'): r = None # 检查USE_PROXY，防止proxies单独起作用
if r is None:
print亮红('[PROXY] 网络代理状态：未配置。无代理状态下很可能无法访问OpenAI家族的模型。建议：检查USE_PROXY选项是否修改。')
log亮红('[PROXY] 网络代理状态：未配置。无代理状态下很可能无法访问OpenAI家族的模型。建议：检查USE_PROXY选项是否修改。')
else:
print亮绿('[PROXY] 网络代理状态：已配置。配置信息如下：', r)
log亮绿('[PROXY] 网络代理状态：已配置。配置信息如下：', str(r))
assert isinstance(r, dict), 'proxies格式错误，请注意proxies选项的格式，不要遗漏括号。'
return r

17
shared_utils/cookie_manager.py
查看文件

@@ -90,23 +90,6 @@ def make_history_cache():
# """
# with gr.Row():
# txt = gr.Textbox(show_label=False, placeholder="Input question here.", elem_id='user_input_main').style(container=False)
# txtx = gr.Textbox(show_label=False, placeholder="Input question here.", elem_id='user_input_main').style(container=False)
# with gr.Row():
# btn_value = "Test"
# elem_id = "TestCase"
# variant = "primary"
# input_list = [txt, txtx]
# output_list = [txt, txtx]
# input_name_list = ["txt(input)", "txtx(input)"]
# output_name_list = ["txt", "txtx"]
# js_callback = """(txt, txtx)=>{console.log(txt); console.log(txtx);}"""
# def function(txt, txtx):
# return "booo", "goooo"
# create_button_with_javascript_callback(btn_value, elem_id, variant, js_callback, input_list, output_list, function, input_name_list, output_name_list)
# """
def create_button_with_javascript_callback(btn_value, elem_id, variant, js_callback, input_list, output_list, function, input_name_list, output_name_list):
import gradio as gr
middle_ware_component = gr.Textbox(visible=False, elem_id=elem_id+'_buffer')

13
shared_utils/handle_upload.py
查看文件

@@ -8,6 +8,7 @@ import gradio
import shutil
import glob
from shared_utils.config_loader import get_conf
from loguru import logger
def html_local_file(file):
base_path = os.path.dirname(__file__) # 项目目录
@@ -100,7 +101,7 @@ def extract_archive(file_path, dest_dir):
with zipfile.ZipFile(file_path, "r") as zipobj:
zipobj._extract_member = lambda a,b,c: zip_extract_member_new(zipobj, a,b,c) # 修复中文乱码的问题
zipobj.extractall(path=dest_dir)
print("Successfully extracted zip archive to {}".format(dest_dir))
logger.info("Successfully extracted zip archive to {}".format(dest_dir))
elif file_extension in [".tar", ".gz", ".bz2"]:
with tarfile.open(file_path, "r:*") as tarobj:
@@ -113,7 +114,7 @@ def extract_archive(file_path, dest_dir):
raise Exception(f"Attempted Path Traversal in {member.name}")
tarobj.extractall(path=dest_dir)
print("Successfully extracted tar archive to {}".format(dest_dir))
logger.info("Successfully extracted tar archive to {}".format(dest_dir))
# 第三方库，需要预先pip install rarfile
# 此外，Windows上还需要安装winrar软件，配置其Path环境变量，如"C:\Program Files\WinRAR"才可以
@@ -123,9 +124,9 @@ def extract_archive(file_path, dest_dir):
with rarfile.RarFile(file_path) as rf:
rf.extractall(path=dest_dir)
print("Successfully extracted rar archive to {}".format(dest_dir))
logger.info("Successfully extracted rar archive to {}".format(dest_dir))
except:
print("Rar format requires additional dependencies to install")
logger.info("Rar format requires additional dependencies to install")
return "\n\n解压失败! 需要安装pip install rarfile来解压rar文件。建议：使用zip压缩格式。"
# 第三方库，需要预先pip install py7zr
@@ -135,9 +136,9 @@ def extract_archive(file_path, dest_dir):
with py7zr.SevenZipFile(file_path, mode="r") as f:
f.extractall(path=dest_dir)
print("Successfully extracted 7z archive to {}".format(dest_dir))
logger.info("Successfully extracted 7z archive to {}".format(dest_dir))
except:
print("7z format requires additional dependencies to install")
logger.info("7z format requires additional dependencies to install")
return "\n\n解压失败! 需要安装pip install py7zr来解压7z文件"
else:
return ""

7
shared_utils/key_pattern_manager.py
查看文件

@@ -34,6 +34,9 @@ def is_api2d_key(key):
API_MATCH_API2D = re.match(r"fk[a-zA-Z0-9]{6}-[a-zA-Z0-9]{32}$", key)
return bool(API_MATCH_API2D)
def is_openroute_api_key(key):
API_MATCH_OPENROUTE = re.match(r"sk-or-v1-[a-zA-Z0-9]{64}$", key)
return bool(API_MATCH_OPENROUTE)
def is_cohere_api_key(key):
API_MATCH_AZURE = re.match(r"[a-zA-Z0-9]{40}$", key)
@@ -89,6 +92,10 @@ def select_api_key(keys, llm_model):
if llm_model.startswith('cohere-'):
for k in key_list:
if is_cohere_api_key(k): avail_key_list.append(k)
if llm_model.startswith('openrouter-'):
for k in key_list:
if is_openroute_api_key(k): avail_key_list.append(k)
if len(avail_key_list) == 0:
raise RuntimeError(f"您提供的api-key不满足要求，不包含任何可用于{llm_model}的api-key。您可能选择了错误的模型或请求源（左上角更换模型菜单中可切换openai,azure,claude,cohere等请求源）。")

69
shared_utils/logging.py 普通文件
查看文件

@@ -0,0 +1,69 @@
from loguru import logger
import logging
import sys
import os
def chat_log_filter(record):
return "chat_msg" in record["extra"]
def not_chat_log_filter(record):
return "chat_msg" not in record["extra"]
def formatter_with_clip(record):
# Note this function returns the string to be formatted, not the actual message to be logged
# record["extra"]["serialized"] = "555555"
max_len = 12
record['function_x'] = record['function'].center(max_len)
if len(record['function_x']) > max_len:
record['function_x'] = ".." + record['function_x'][-(max_len-2):]
record['line_x'] = str(record['line']).ljust(3)
return '<green>{time:HH:mm}</green> | <cyan>{function_x}</cyan>:<cyan>{line_x}</cyan> | <level>{message}</level>\n'
def setup_logging(PATH_LOGGING):
admin_log_path = os.path.join(PATH_LOGGING, "admin")
os.makedirs(admin_log_path, exist_ok=True)
sensitive_log_path = os.path.join(admin_log_path, "chat_secrets.log")
regular_log_path = os.path.join(admin_log_path, "console_log.log")
logger.remove()
logger.configure(
levels=[dict(name="WARNING", color="<g>")],
)
logger.add(
sys.stderr,
format=formatter_with_clip,
# format='<green>{time:HH:mm}</green> | <cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>',
filter=(lambda record: not chat_log_filter(record)),
colorize=True,
enqueue=True
)
logger.add(
sensitive_log_path,
format='<green>{time:MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>',
rotation="10 MB",
filter=chat_log_filter,
enqueue=True,
)
logger.add(
regular_log_path,
format='<green>{time:MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>',
rotation="10 MB",
filter=not_chat_log_filter,
enqueue=True,
)
logging.getLogger("httpx").setLevel(logging.WARNING)
logger.warning(f"所有对话记录将自动保存在本地目录{sensitive_log_path}, 请注意自我隐私保护哦！")
# logger.bind(chat_msg=True).info("This message is logged to the file!")
# logger.debug(f"debug message")
# logger.info(f"info message")
# logger.success(f"success message")
# logger.error(f"error message")
# logger.add("special.log", filter=lambda record: "special" in record["extra"])
# logger.debug("This message is not logged to the file")