int/gpt_academic
1
0
派生 0
镜像自地址 https://github.com/binary-husky/gpt_academic.git 已同步 2025-12-06 06:26:47 +00:00
代码 工单 软件包 项目 版本发布 百科 动态

Merge branch 'GHSA-3jrq-66fm-w7xr' into frontier

浏览代码
这个提交包含在：
binary-husky
2024-06-18 10:19:10 +00:00
父节点 1e16485087 47cedde954
当前提交 9dfbff7fd0
共有 7 个文件被更改，包括 132 次插入 和 35 次删除
下载 Patch 文件 下载 Diff 文件 展开所有文件 折叠所有文件

4
.gitignore vendored
查看文件

@@ -153,4 +153,6 @@ media
flagged flagged
request_llms/ChatGLM-6b-onnx-u8s8 request_llms/ChatGLM-6b-onnx-u8s8
.pre-commit-config.yaml .pre-commit-config.yaml
themes/common.js.min.*.js themes/common.js.min.*.js
test*
objdump*

98
crazy_functions/Conversation_To_File.py
查看文件

@@ -10,27 +10,61 @@ def write_chat_to_file(chatbot, history=None, file_name=None):
""" """
import os import os
import time import time
from themes.theme import advanced_css
if file_name is None: if file_name is None:
file_name = f_prefix + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '.html' file_name = f_prefix + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '.html'
fp = os.path.join(get_log_folder(get_user(chatbot), plugin_name='chat_history'), file_name) fp = os.path.join(get_log_folder(get_user(chatbot), plugin_name='chat_history'), file_name)
with open(fp, 'w', encoding='utf8') as f: with open(fp, 'w', encoding='utf8') as f:
from themes.theme import advanced_css from textwrap import dedent
f.write(f'<!DOCTYPE html><head><meta charset="utf-8"><title>对话历史</title><style>{advanced_css}</style></head>') form = dedent("""
<!DOCTYPE html><head><meta charset="utf-8"><title>对话存档</title><style>{CSS}</style></head>
<body>
<div class="test_temp1" style="width:10%; height: 500px; float:left;"></div>
<div class="test_temp2" style="width:80%;padding: 40px;float:left;padding-left: 20px;padding-right: 20px;box-shadow: rgba(0, 0, 0, 0.2) 0px 0px 8px 8px;border-radius: 10px;">
<div class="chat-body" style="display: flex;justify-content: center;flex-direction: column;align-items: center;flex-wrap: nowrap;">
{CHAT_PREVIEW}
<div></div>
<div></div>
<div style="text-align: center;width:80%;padding: 0px;float:left;padding-left:20px;padding-right:20px;box-shadow: rgba(0, 0, 0, 0.05) 0px 0px 1px 2px;border-radius: 1px;">对话（原始数据）</div>
{HISTORY_PREVIEW}
</div>
</div>
<div class="test_temp3" style="width:10%; height: 500px; float:left;"></div>
</body>
""")
qa_from = dedent("""
<div class="QaBox" style="width:80%;padding: 20px;margin-bottom: 20px;box-shadow: rgb(0 255 159 / 50%) 0px 0px 1px 2px;border-radius: 4px;">
<div class="Question" style="border-radius: 2px;">{QUESTION}</div>
<hr color="blue" style="border-top: dotted 2px #ccc;">
<div class="Answer" style="border-radius: 2px;">{ANSWER}</div>
</div>
""")
history_from = dedent("""
<div class="historyBox" style="width:80%;padding: 0px;float:left;padding-left:20px;padding-right:20px;box-shadow: rgba(0, 0, 0, 0.05) 0px 0px 1px 2px;border-radius: 1px;">
<div class="entry" style="border-radius: 2px;">{ENTRY}</div>
</div>
""")
CHAT_PREVIEW_BUF = ""
for i, contents in enumerate(chatbot): for i, contents in enumerate(chatbot):
for j, content in enumerate(contents): question, answer = contents[0], contents[1]
try: # 这个bug没找到触发条件，暂时先这样顶一下 if question is None: question = ""
if type(content) != str: content = str(content) try: question = str(question)
except: except: question = ""
continue if answer is None: answer = ""
f.write(content) try: answer = str(answer)
if j == 0: except: answer = ""
f.write('<hr style="border-top: dotted 3px #ccc;">') CHAT_PREVIEW_BUF += qa_from.format(QUESTION=question, ANSWER=answer)
f.write('<hr color="red"> \n\n')
f.write('<hr color="blue"> \n\n raw chat context:\n') HISTORY_PREVIEW_BUF = ""
f.write('<code>')
for h in history: for h in history:
f.write("\n>>>" + h) HISTORY_PREVIEW_BUF += history_from.format(ENTRY=h)
f.write('</code>') html_content = form.format(CHAT_PREVIEW=CHAT_PREVIEW_BUF, HISTORY_PREVIEW=HISTORY_PREVIEW_BUF, CSS=advanced_css)
f.write(html_content)
promote_file_to_downloadzone(fp, rename_file=file_name, chatbot=chatbot) promote_file_to_downloadzone(fp, rename_file=file_name, chatbot=chatbot)
return '对话历史写入：' + fp return '对话历史写入：' + fp
@@ -41,7 +75,7 @@ def gen_file_preview(file_name):
# pattern to match the text between <head> and </head> # pattern to match the text between <head> and </head>
pattern = re.compile(r'<head>.*?</head>', flags=re.DOTALL) pattern = re.compile(r'<head>.*?</head>', flags=re.DOTALL)
file_content = re.sub(pattern, '', file_content) file_content = re.sub(pattern, '', file_content)
html, history = file_content.split('<hr color="blue"> \n\n raw chat context:\n') html, history = file_content.split('<hr color="blue"> \n\n 对话数据 (无渲染):\n')
history = history.strip('<code>') history = history.strip('<code>')
history = history.strip('</code>') history = history.strip('</code>')
history = history.split("\n>>>") history = history.split("\n>>>")
@@ -52,21 +86,25 @@ def gen_file_preview(file_name):
def read_file_to_chat(chatbot, history, file_name): def read_file_to_chat(chatbot, history, file_name):
with open(file_name, 'r', encoding='utf8') as f: with open(file_name, 'r', encoding='utf8') as f:
file_content = f.read() file_content = f.read()
# pattern to match the text between <head> and </head> from bs4 import BeautifulSoup
pattern = re.compile(r'<head>.*?</head>', flags=re.DOTALL) soup = BeautifulSoup(file_content, 'lxml')
file_content = re.sub(pattern, '', file_content) # 提取QaBox信息
html, history = file_content.split('<hr color="blue"> \n\n raw chat context:\n')
history = history.strip('<code>')
history = history.strip('</code>')
history = history.split("\n>>>")
history = list(filter(lambda x:x!="", history))
html = html.split('<hr color="red"> \n\n')
html = list(filter(lambda x:x!="", html))
chatbot.clear() chatbot.clear()
for i, h in enumerate(html): qa_box_list = []
i_say, gpt_say = h.split('<hr style="border-top: dotted 3px #ccc;">') qa_boxes = soup.find_all("div", class_="QaBox")
chatbot.append([i_say, gpt_say]) for box in qa_boxes:
chatbot.append([f"存档文件详情？", f"[Local Message] 载入对话{len(html)}条，上下文{len(history)}条。"]) question = box.find("div", class_="Question").get_text(strip=False)
answer = box.find("div", class_="Answer").get_text(strip=False)
qa_box_list.append({"Question": question, "Answer": answer})
chatbot.append([question, answer])
# 提取historyBox信息
history_box_list = []
history_boxes = soup.find_all("div", class_="historyBox")
for box in history_boxes:
entry = box.find("div", class_="entry").get_text(strip=False)
history_box_list.append(entry)
history = history_box_list
chatbot.append([None, f"[Local Message] 载入对话{len(qa_box_list)}条，上下文{len(history)}条。"])
return chatbot, history return chatbot, history
@CatchException @CatchException

6
crazy_functions/Latex_Function.py
查看文件

@@ -271,6 +271,8 @@ def Latex英文纠错加PDF对比(txt, llm_kwargs, plugin_kwargs, chatbot, histo
project_folder = desend_to_extracted_folder_if_exist(project_folder) project_folder = desend_to_extracted_folder_if_exist(project_folder)
# <-------------- move latex project away from temp folder -------------> # <-------------- move latex project away from temp folder ------------->
from shared_utils.fastapi_server import validate_path_safety
validate_path_safety(project_folder, chatbot.get_user())
project_folder = move_project(project_folder, arxiv_id=None) project_folder = move_project(project_folder, arxiv_id=None)
# <-------------- if merge_translate_zh is already generated, skip gpt req -------------> # <-------------- if merge_translate_zh is already generated, skip gpt req ------------->
@@ -365,6 +367,8 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot,
project_folder = desend_to_extracted_folder_if_exist(project_folder) project_folder = desend_to_extracted_folder_if_exist(project_folder)
# <-------------- move latex project away from temp folder -------------> # <-------------- move latex project away from temp folder ------------->
from shared_utils.fastapi_server import validate_path_safety
validate_path_safety(project_folder, chatbot.get_user())
project_folder = move_project(project_folder, arxiv_id) project_folder = move_project(project_folder, arxiv_id)
# <-------------- if merge_translate_zh is already generated, skip gpt req -------------> # <-------------- if merge_translate_zh is already generated, skip gpt req ------------->
@@ -503,6 +507,8 @@ def PDF翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot, h
project_folder = desend_to_extracted_folder_if_exist(project_folder) project_folder = desend_to_extracted_folder_if_exist(project_folder)
# <-------------- move latex project away from temp folder -------------> # <-------------- move latex project away from temp folder ------------->
from shared_utils.fastapi_server import validate_path_safety
validate_path_safety(project_folder, chatbot.get_user())
project_folder = move_project(project_folder) project_folder = move_project(project_folder)
# <-------------- set a hash tag for repeat-checking -------------> # <-------------- set a hash tag for repeat-checking ------------->

14
crazy_functions/解析项目源代码.py
查看文件

@@ -1,6 +1,7 @@
from toolbox import update_ui, promote_file_to_downloadzone, disable_auto_promotion from toolbox import update_ui, promote_file_to_downloadzone, disable_auto_promotion
from toolbox import CatchException, report_exception, write_history_to_file from toolbox import CatchException, report_exception, write_history_to_file
from .crazy_utils import input_clipping from shared_utils.fastapi_server import validate_path_safety
from crazy_functions.crazy_utils import input_clipping
def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt): def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt):
import os, copy import os, copy
@@ -128,6 +129,7 @@ def 解析一个Python项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, s
import glob, os import glob, os
if os.path.exists(txt): if os.path.exists(txt):
project_folder = txt project_folder = txt
validate_path_safety(project_folder, chatbot.get_user())
else: else:
if txt == "": txt = '空空如也的输入栏' if txt == "": txt = '空空如也的输入栏'
report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
@@ -146,6 +148,7 @@ def 解析一个Matlab项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, s
import glob, os import glob, os
if os.path.exists(txt): if os.path.exists(txt):
project_folder = txt project_folder = txt
validate_path_safety(project_folder, chatbot.get_user())
else: else:
if txt == "": txt = '空空如也的输入栏' if txt == "": txt = '空空如也的输入栏'
report_exception(chatbot, history, a = f"解析Matlab项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") report_exception(chatbot, history, a = f"解析Matlab项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
@@ -164,6 +167,7 @@ def 解析一个C项目的头文件(txt, llm_kwargs, plugin_kwargs, chatbot, his
import glob, os import glob, os
if os.path.exists(txt): if os.path.exists(txt):
project_folder = txt project_folder = txt
validate_path_safety(project_folder, chatbot.get_user())
else: else:
if txt == "": txt = '空空如也的输入栏' if txt == "": txt = '空空如也的输入栏'
report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
@@ -184,6 +188,7 @@ def 解析一个C项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, system
import glob, os import glob, os
if os.path.exists(txt): if os.path.exists(txt):
project_folder = txt project_folder = txt
validate_path_safety(project_folder, chatbot.get_user())
else: else:
if txt == "": txt = '空空如也的输入栏' if txt == "": txt = '空空如也的输入栏'
report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
@@ -206,6 +211,7 @@ def 解析一个Java项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys
import glob, os import glob, os
if os.path.exists(txt): if os.path.exists(txt):
project_folder = txt project_folder = txt
validate_path_safety(project_folder, chatbot.get_user())
else: else:
if txt == "": txt = '空空如也的输入栏' if txt == "": txt = '空空如也的输入栏'
report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到本地项目或无权访问: {txt}") report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到本地项目或无权访问: {txt}")
@@ -228,6 +234,7 @@ def 解析一个前端项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, s
import glob, os import glob, os
if os.path.exists(txt): if os.path.exists(txt):
project_folder = txt project_folder = txt
validate_path_safety(project_folder, chatbot.get_user())
else: else:
if txt == "": txt = '空空如也的输入栏' if txt == "": txt = '空空如也的输入栏'
report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到本地项目或无权访问: {txt}") report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到本地项目或无权访问: {txt}")
@@ -257,6 +264,7 @@ def 解析一个Golang项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, s
import glob, os import glob, os
if os.path.exists(txt): if os.path.exists(txt):
project_folder = txt project_folder = txt
validate_path_safety(project_folder, chatbot.get_user())
else: else:
if txt == "": txt = '空空如也的输入栏' if txt == "": txt = '空空如也的输入栏'
report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到本地项目或无权访问: {txt}") report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到本地项目或无权访问: {txt}")
@@ -278,6 +286,7 @@ def 解析一个Rust项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys
import glob, os import glob, os
if os.path.exists(txt): if os.path.exists(txt):
project_folder = txt project_folder = txt
validate_path_safety(project_folder, chatbot.get_user())
else: else:
if txt == "": txt = '空空如也的输入栏' if txt == "": txt = '空空如也的输入栏'
report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到本地项目或无权访问: {txt}") report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到本地项目或无权访问: {txt}")
@@ -298,6 +307,7 @@ def 解析一个Lua项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, syst
import glob, os import glob, os
if os.path.exists(txt): if os.path.exists(txt):
project_folder = txt project_folder = txt
validate_path_safety(project_folder, chatbot.get_user())
else: else:
if txt == "": txt = '空空如也的输入栏' if txt == "": txt = '空空如也的输入栏'
report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
@@ -320,6 +330,7 @@ def 解析一个CSharp项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, s
import glob, os import glob, os
if os.path.exists(txt): if os.path.exists(txt):
project_folder = txt project_folder = txt
validate_path_safety(project_folder, chatbot.get_user())
else: else:
if txt == "": txt = '空空如也的输入栏' if txt == "": txt = '空空如也的输入栏'
report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
@@ -357,6 +368,7 @@ def 解析任意code项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys
import glob, os, re import glob, os, re
if os.path.exists(txt): if os.path.exists(txt):
project_folder = txt project_folder = txt
validate_path_safety(project_folder, chatbot.get_user())
else: else:
if txt == "": txt = '空空如也的输入栏' if txt == "": txt = '空空如也的输入栏'
report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")

2
shared_utils/advanced_markdown_format.py
查看文件

@@ -271,7 +271,7 @@ def markdown_convertion_for_file(txt):
""" """
from themes.theme import advanced_css from themes.theme import advanced_css
pre = f""" pre = f"""
<!DOCTYPE html><head><meta charset="utf-8"><title>对话历史</title><style>{advanced_css}</style></head> <!DOCTYPE html><head><meta charset="utf-8"><title>PDF文档翻译</title><style>{advanced_css}</style></head>
<body> <body>
<div class="test_temp1" style="width:10%; height: 500px; float:left;"></div> <div class="test_temp1" style="width:10%; height: 500px; float:left;"></div>
<div class="test_temp2" style="width:80%;padding: 40px;float:left;padding-left: 20px;padding-right: 20px;box-shadow: rgba(0, 0, 0, 0.2) 0px 0px 8px 8px;border-radius: 10px;"> <div class="test_temp2" style="width:80%;padding: 40px;float:left;padding-left: 20px;padding-right: 20px;box-shadow: rgba(0, 0, 0, 0.2) 0px 0px 8px 8px;border-radius: 10px;">

22
shared_utils/fastapi_server.py
查看文件

@@ -47,6 +47,28 @@ queue cocurrent effectiveness
import os, requests, threading, time import os, requests, threading, time
import uvicorn import uvicorn
def validate_path_safety(path_or_url, user):
from toolbox import get_conf, default_user_name
from toolbox import FriendlyException
PATH_PRIVATE_UPLOAD, PATH_LOGGING = get_conf('PATH_PRIVATE_UPLOAD', 'PATH_LOGGING')
sensitive_path = None
path_or_url = os.path.relpath(path_or_url)
if path_or_url.startswith(PATH_LOGGING): # 日志文件（按用户划分）
sensitive_path = PATH_LOGGING
elif path_or_url.startswith(PATH_PRIVATE_UPLOAD): # 用户的上传目录（按用户划分）
sensitive_path = PATH_PRIVATE_UPLOAD
elif path_or_url.startswith('tests'): # 一个常用的测试目录
return True
else:
raise FriendlyException(f"输入文件的路径 ({path_or_url}) 存在，但位置非法。请将文件上传后再执行该任务。") # return False
if sensitive_path:
allowed_users = [user, 'autogen', default_user_name] # three user path that can be accessed
for user_allowed in allowed_users:
if f"{os.sep}".join(path_or_url.split(os.sep)[:2]) == os.path.join(sensitive_path, user_allowed):
return True
raise FriendlyException(f"输入文件的路径 ({path_or_url}) 存在，但属于其他用户。请将文件上传后再执行该任务。") # return False
return True
def _authorize_user(path_or_url, request, gradio_app): def _authorize_user(path_or_url, request, gradio_app):
from toolbox import get_conf, default_user_name from toolbox import get_conf, default_user_name
PATH_PRIVATE_UPLOAD, PATH_LOGGING = get_conf('PATH_PRIVATE_UPLOAD', 'PATH_LOGGING') PATH_PRIVATE_UPLOAD, PATH_LOGGING = get_conf('PATH_PRIVATE_UPLOAD', 'PATH_LOGGING')

21
toolbox.py
查看文件

@@ -10,6 +10,7 @@ import glob
import logging import logging
import uuid import uuid
from functools import wraps from functools import wraps
from textwrap import dedent
from shared_utils.config_loader import get_conf from shared_utils.config_loader import get_conf
from shared_utils.config_loader import set_conf from shared_utils.config_loader import set_conf
from shared_utils.config_loader import set_multi_conf from shared_utils.config_loader import set_multi_conf
@@ -193,9 +194,20 @@ def trimmed_format_exc():
replace_path = "." replace_path = "."
return str.replace(current_path, replace_path) return str.replace(current_path, replace_path)
def trimmed_format_exc_markdown(): def trimmed_format_exc_markdown():
return '\n\n```\n' + trimmed_format_exc() + '```' return '\n\n```\n' + trimmed_format_exc() + '```'
class FriendlyException(Exception):
def generate_error_html(self):
return dedent(f"""
<div class="center-div" style="color: crimson;text-align: center;">
{"<br>".join(self.args)}
</div>
""")
def CatchException(f): def CatchException(f):
""" """
装饰器函数，捕捉函数f中的异常并封装到一个生成器中返回，并显示到聊天当中。 装饰器函数，捕捉函数f中的异常并封装到一个生成器中返回，并显示到聊天当中。
@@ -206,13 +218,18 @@ def CatchException(f):
chatbot_with_cookie:ChatBotWithCookies, history:list, *args, **kwargs): chatbot_with_cookie:ChatBotWithCookies, history:list, *args, **kwargs):
try: try:
yield from f(main_input, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, *args, **kwargs) yield from f(main_input, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, *args, **kwargs)
except FriendlyException as e:
if len(chatbot_with_cookie) == 0:
chatbot_with_cookie.clear()
chatbot_with_cookie.append(["插件调度异常", None])
chatbot_with_cookie[-1] = [chatbot_with_cookie[-1][0], e.generate_error_html()]
yield from update_ui(chatbot=chatbot_with_cookie, history=history, msg=f'异常') # 刷新界面
except Exception as e: except Exception as e:
from toolbox import get_conf
tb_str = '```\n' + trimmed_format_exc() + '```' tb_str = '```\n' + trimmed_format_exc() + '```'
if len(chatbot_with_cookie) == 0: if len(chatbot_with_cookie) == 0:
chatbot_with_cookie.clear() chatbot_with_cookie.clear()
chatbot_with_cookie.append(["插件调度异常", "异常原因"]) chatbot_with_cookie.append(["插件调度异常", "异常原因"])
chatbot_with_cookie[-1] = (chatbot_with_cookie[-1][0], f"[Local Message] 插件调用出错: \n\n{tb_str} \n") chatbot_with_cookie[-1] = [chatbot_with_cookie[-1][0], f"[Local Message] 插件调用出错: \n\n{tb_str} \n"]
yield from update_ui(chatbot=chatbot_with_cookie, history=history, msg=f'异常 {e}') # 刷新界面 yield from update_ui(chatbot=chatbot_with_cookie, history=history, msg=f'异常 {e}') # 刷新界面
return decorated return decorated