镜像自地址
https://github.com/binary-husky/gpt_academic.git
已同步 2025-12-06 06:26:47 +00:00
Merge pull request from GHSA-3jrq-66fm-w7xr
这个提交包含在：
iluem
2
.gitignore
vendored
2
.gitignore
vendored
@@ -154,3 +154,5 @@ flagged
|
||||
request_llms/ChatGLM-6b-onnx-u8s8
|
||||
.pre-commit-config.yaml
|
||||
themes/common.js.min.*.js
|
||||
test*
|
||||
objdump*
|
||||
@@ -10,27 +10,61 @@ def write_chat_to_file(chatbot, history=None, file_name=None):
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
from themes.theme import advanced_css
|
||||
|
||||
if file_name is None:
|
||||
file_name = f_prefix + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '.html'
|
||||
fp = os.path.join(get_log_folder(get_user(chatbot), plugin_name='chat_history'), file_name)
|
||||
|
||||
with open(fp, 'w', encoding='utf8') as f:
|
||||
from themes.theme import advanced_css
|
||||
f.write(f'<!DOCTYPE html><head><meta charset="utf-8"><title>对话历史</title><style>{advanced_css}</style></head>')
|
||||
from textwrap import dedent
|
||||
form = dedent("""
|
||||
<!DOCTYPE html><head><meta charset="utf-8"><title>对话存档</title><style>{CSS}</style></head>
|
||||
<body>
|
||||
<div class="test_temp1" style="width:10%; height: 500px; float:left;"></div>
|
||||
<div class="test_temp2" style="width:80%;padding: 40px;float:left;padding-left: 20px;padding-right: 20px;box-shadow: rgba(0, 0, 0, 0.2) 0px 0px 8px 8px;border-radius: 10px;">
|
||||
<div class="chat-body" style="display: flex;justify-content: center;flex-direction: column;align-items: center;flex-wrap: nowrap;">
|
||||
{CHAT_PREVIEW}
|
||||
<div></div>
|
||||
<div></div>
|
||||
<div style="text-align: center;width:80%;padding: 0px;float:left;padding-left:20px;padding-right:20px;box-shadow: rgba(0, 0, 0, 0.05) 0px 0px 1px 2px;border-radius: 1px;">对话（原始数据）</div>
|
||||
{HISTORY_PREVIEW}
|
||||
</div>
|
||||
</div>
|
||||
<div class="test_temp3" style="width:10%; height: 500px; float:left;"></div>
|
||||
</body>
|
||||
""")
|
||||
|
||||
qa_from = dedent("""
|
||||
<div class="QaBox" style="width:80%;padding: 20px;margin-bottom: 20px;box-shadow: rgb(0 255 159 / 50%) 0px 0px 1px 2px;border-radius: 4px;">
|
||||
<div class="Question" style="border-radius: 2px;">{QUESTION}</div>
|
||||
<hr color="blue" style="border-top: dotted 2px #ccc;">
|
||||
<div class="Answer" style="border-radius: 2px;">{ANSWER}</div>
|
||||
</div>
|
||||
""")
|
||||
|
||||
history_from = dedent("""
|
||||
<div class="historyBox" style="width:80%;padding: 0px;float:left;padding-left:20px;padding-right:20px;box-shadow: rgba(0, 0, 0, 0.05) 0px 0px 1px 2px;border-radius: 1px;">
|
||||
<div class="entry" style="border-radius: 2px;">{ENTRY}</div>
|
||||
</div>
|
||||
""")
|
||||
CHAT_PREVIEW_BUF = ""
|
||||
for i, contents in enumerate(chatbot):
|
||||
for j, content in enumerate(contents):
|
||||
try: # 这个bug没找到触发条件，暂时先这样顶一下
|
||||
if type(content) != str: content = str(content)
|
||||
except:
|
||||
continue
|
||||
f.write(content)
|
||||
if j == 0:
|
||||
f.write('<hr style="border-top: dotted 3px #ccc;">')
|
||||
f.write('<hr color="red"> \n\n')
|
||||
f.write('<hr color="blue"> \n\n raw chat context:\n')
|
||||
f.write('<code>')
|
||||
question, answer = contents[0], contents[1]
|
||||
if question is None: question = ""
|
||||
try: question = str(question)
|
||||
except: question = ""
|
||||
if answer is None: answer = ""
|
||||
try: answer = str(answer)
|
||||
except: answer = ""
|
||||
CHAT_PREVIEW_BUF += qa_from.format(QUESTION=question, ANSWER=answer)
|
||||
|
||||
HISTORY_PREVIEW_BUF = ""
|
||||
for h in history:
|
||||
f.write("\n>>>" + h)
|
||||
f.write('</code>')
|
||||
HISTORY_PREVIEW_BUF += history_from.format(ENTRY=h)
|
||||
html_content = form.format(CHAT_PREVIEW=CHAT_PREVIEW_BUF, HISTORY_PREVIEW=HISTORY_PREVIEW_BUF, CSS=advanced_css)
|
||||
f.write(html_content)
|
||||
|
||||
promote_file_to_downloadzone(fp, rename_file=file_name, chatbot=chatbot)
|
||||
return '对话历史写入：' + fp
|
||||
|
||||
@@ -41,7 +75,7 @@ def gen_file_preview(file_name):
|
||||
# pattern to match the text between <head> and </head>
|
||||
pattern = re.compile(r'<head>.*?</head>', flags=re.DOTALL)
|
||||
file_content = re.sub(pattern, '', file_content)
|
||||
html, history = file_content.split('<hr color="blue"> \n\n raw chat context:\n')
|
||||
html, history = file_content.split('<hr color="blue"> \n\n 对话数据 (无渲染):\n')
|
||||
history = history.strip('<code>')
|
||||
history = history.strip('</code>')
|
||||
history = history.split("\n>>>")
|
||||
@@ -52,21 +86,25 @@ def gen_file_preview(file_name):
|
||||
def read_file_to_chat(chatbot, history, file_name):
|
||||
with open(file_name, 'r', encoding='utf8') as f:
|
||||
file_content = f.read()
|
||||
# pattern to match the text between <head> and </head>
|
||||
pattern = re.compile(r'<head>.*?</head>', flags=re.DOTALL)
|
||||
file_content = re.sub(pattern, '', file_content)
|
||||
html, history = file_content.split('<hr color="blue"> \n\n raw chat context:\n')
|
||||
history = history.strip('<code>')
|
||||
history = history.strip('</code>')
|
||||
history = history.split("\n>>>")
|
||||
history = list(filter(lambda x:x!="", history))
|
||||
html = html.split('<hr color="red"> \n\n')
|
||||
html = list(filter(lambda x:x!="", html))
|
||||
from bs4 import BeautifulSoup
|
||||
soup = BeautifulSoup(file_content, 'lxml')
|
||||
# 提取QaBox信息
|
||||
chatbot.clear()
|
||||
for i, h in enumerate(html):
|
||||
i_say, gpt_say = h.split('<hr style="border-top: dotted 3px #ccc;">')
|
||||
chatbot.append([i_say, gpt_say])
|
||||
chatbot.append([f"存档文件详情？", f"[Local Message] 载入对话{len(html)}条，上下文{len(history)}条。"])
|
||||
qa_box_list = []
|
||||
qa_boxes = soup.find_all("div", class_="QaBox")
|
||||
for box in qa_boxes:
|
||||
question = box.find("div", class_="Question").get_text(strip=False)
|
||||
answer = box.find("div", class_="Answer").get_text(strip=False)
|
||||
qa_box_list.append({"Question": question, "Answer": answer})
|
||||
chatbot.append([question, answer])
|
||||
# 提取historyBox信息
|
||||
history_box_list = []
|
||||
history_boxes = soup.find_all("div", class_="historyBox")
|
||||
for box in history_boxes:
|
||||
entry = box.find("div", class_="entry").get_text(strip=False)
|
||||
history_box_list.append(entry)
|
||||
history = history_box_list
|
||||
chatbot.append([None, f"[Local Message] 载入对话{len(qa_box_list)}条，上下文{len(history)}条。"])
|
||||
return chatbot, history
|
||||
|
||||
@CatchException
|
||||
@@ -180,5 +218,3 @@ def 删除所有本地对话历史记录(txt, llm_kwargs, plugin_kwargs, chatbot
|
||||
chatbot.append([f"删除所有历史对话文件", f"已删除<br/>{local_history}"])
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
return
|
||||
|
||||
|
||||
|
||||
@@ -271,6 +271,8 @@ def Latex英文纠错加PDF对比(txt, llm_kwargs, plugin_kwargs, chatbot, histo
|
||||
project_folder = desend_to_extracted_folder_if_exist(project_folder)
|
||||
|
||||
# <-------------- move latex project away from temp folder ------------->
|
||||
from shared_utils.fastapi_server import validate_path_safety
|
||||
validate_path_safety(project_folder, chatbot.get_user())
|
||||
project_folder = move_project(project_folder, arxiv_id=None)
|
||||
|
||||
# <-------------- if merge_translate_zh is already generated, skip gpt req ------------->
|
||||
@@ -365,6 +367,8 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot,
|
||||
project_folder = desend_to_extracted_folder_if_exist(project_folder)
|
||||
|
||||
# <-------------- move latex project away from temp folder ------------->
|
||||
from shared_utils.fastapi_server import validate_path_safety
|
||||
validate_path_safety(project_folder, chatbot.get_user())
|
||||
project_folder = move_project(project_folder, arxiv_id)
|
||||
|
||||
# <-------------- if merge_translate_zh is already generated, skip gpt req ------------->
|
||||
@@ -503,6 +507,8 @@ def PDF翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot, h
|
||||
project_folder = desend_to_extracted_folder_if_exist(project_folder)
|
||||
|
||||
# <-------------- move latex project away from temp folder ------------->
|
||||
from shared_utils.fastapi_server import validate_path_safety
|
||||
validate_path_safety(project_folder, chatbot.get_user())
|
||||
project_folder = move_project(project_folder)
|
||||
|
||||
# <-------------- set a hash tag for repeat-checking ------------->
|
||||
|
||||
@@ -8,8 +8,8 @@ class SafeUnpickler(pickle.Unpickler):
|
||||
# 定义允许的安全类
|
||||
safe_classes = {
|
||||
# 在这里添加其他安全的类
|
||||
'LatexPaperFileGroup': LatexPaperFileGroup,
|
||||
'LatexPaperSplit' : LatexPaperSplit,
|
||||
'latex_actions.LatexPaperFileGroup': LatexPaperFileGroup,
|
||||
'latex_actions.LatexPaperSplit' : LatexPaperSplit,
|
||||
}
|
||||
return safe_classes
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from toolbox import update_ui, promote_file_to_downloadzone, disable_auto_promotion
|
||||
from toolbox import CatchException, report_exception, write_history_to_file
|
||||
from .crazy_utils import input_clipping
|
||||
from shared_utils.fastapi_server import validate_path_safety
|
||||
from crazy_functions.crazy_utils import input_clipping
|
||||
|
||||
def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt):
|
||||
import os, copy
|
||||
@@ -128,6 +129,7 @@ def 解析一个Python项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, s
|
||||
import glob, os
|
||||
if os.path.exists(txt):
|
||||
project_folder = txt
|
||||
validate_path_safety(project_folder, chatbot.get_user())
|
||||
else:
|
||||
if txt == "": txt = '空空如也的输入栏'
|
||||
report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
|
||||
@@ -146,6 +148,7 @@ def 解析一个Matlab项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, s
|
||||
import glob, os
|
||||
if os.path.exists(txt):
|
||||
project_folder = txt
|
||||
validate_path_safety(project_folder, chatbot.get_user())
|
||||
else:
|
||||
if txt == "": txt = '空空如也的输入栏'
|
||||
report_exception(chatbot, history, a = f"解析Matlab项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
|
||||
@@ -164,6 +167,7 @@ def 解析一个C项目的头文件(txt, llm_kwargs, plugin_kwargs, chatbot, his
|
||||
import glob, os
|
||||
if os.path.exists(txt):
|
||||
project_folder = txt
|
||||
validate_path_safety(project_folder, chatbot.get_user())
|
||||
else:
|
||||
if txt == "": txt = '空空如也的输入栏'
|
||||
report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
|
||||
@@ -184,6 +188,7 @@ def 解析一个C项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, system
|
||||
import glob, os
|
||||
if os.path.exists(txt):
|
||||
project_folder = txt
|
||||
validate_path_safety(project_folder, chatbot.get_user())
|
||||
else:
|
||||
if txt == "": txt = '空空如也的输入栏'
|
||||
report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
|
||||
@@ -206,6 +211,7 @@ def 解析一个Java项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys
|
||||
import glob, os
|
||||
if os.path.exists(txt):
|
||||
project_folder = txt
|
||||
validate_path_safety(project_folder, chatbot.get_user())
|
||||
else:
|
||||
if txt == "": txt = '空空如也的输入栏'
|
||||
report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到本地项目或无权访问: {txt}")
|
||||
@@ -228,6 +234,7 @@ def 解析一个前端项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, s
|
||||
import glob, os
|
||||
if os.path.exists(txt):
|
||||
project_folder = txt
|
||||
validate_path_safety(project_folder, chatbot.get_user())
|
||||
else:
|
||||
if txt == "": txt = '空空如也的输入栏'
|
||||
report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到本地项目或无权访问: {txt}")
|
||||
@@ -257,6 +264,7 @@ def 解析一个Golang项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, s
|
||||
import glob, os
|
||||
if os.path.exists(txt):
|
||||
project_folder = txt
|
||||
validate_path_safety(project_folder, chatbot.get_user())
|
||||
else:
|
||||
if txt == "": txt = '空空如也的输入栏'
|
||||
report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到本地项目或无权访问: {txt}")
|
||||
@@ -278,6 +286,7 @@ def 解析一个Rust项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys
|
||||
import glob, os
|
||||
if os.path.exists(txt):
|
||||
project_folder = txt
|
||||
validate_path_safety(project_folder, chatbot.get_user())
|
||||
else:
|
||||
if txt == "": txt = '空空如也的输入栏'
|
||||
report_exception(chatbot, history, a=f"解析项目: {txt}", b=f"找不到本地项目或无权访问: {txt}")
|
||||
@@ -298,6 +307,7 @@ def 解析一个Lua项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, syst
|
||||
import glob, os
|
||||
if os.path.exists(txt):
|
||||
project_folder = txt
|
||||
validate_path_safety(project_folder, chatbot.get_user())
|
||||
else:
|
||||
if txt == "": txt = '空空如也的输入栏'
|
||||
report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
|
||||
@@ -320,6 +330,7 @@ def 解析一个CSharp项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, s
|
||||
import glob, os
|
||||
if os.path.exists(txt):
|
||||
project_folder = txt
|
||||
validate_path_safety(project_folder, chatbot.get_user())
|
||||
else:
|
||||
if txt == "": txt = '空空如也的输入栏'
|
||||
report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
|
||||
@@ -357,6 +368,7 @@ def 解析任意code项目(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys
|
||||
import glob, os, re
|
||||
if os.path.exists(txt):
|
||||
project_folder = txt
|
||||
validate_path_safety(project_folder, chatbot.get_user())
|
||||
else:
|
||||
if txt == "": txt = '空空如也的输入栏'
|
||||
report_exception(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
|
||||
|
||||
@@ -213,7 +213,7 @@ def markdown_convertion_for_file(txt):
|
||||
"""
|
||||
from themes.theme import advanced_css
|
||||
pre = f"""
|
||||
<!DOCTYPE html><head><meta charset="utf-8"><title>对话历史</title><style>{advanced_css}</style></head>
|
||||
<!DOCTYPE html><head><meta charset="utf-8"><title>PDF文档翻译</title><style>{advanced_css}</style></head>
|
||||
<body>
|
||||
<div class="test_temp1" style="width:10%; height: 500px; float:left;"></div>
|
||||
<div class="test_temp2" style="width:80%;padding: 40px;float:left;padding-left: 20px;padding-right: 20px;box-shadow: rgba(0, 0, 0, 0.2) 0px 0px 8px 8px;border-radius: 10px;">
|
||||
|
||||
@@ -47,6 +47,28 @@ queue cocurrent effectiveness
|
||||
import os, requests, threading, time
|
||||
import uvicorn
|
||||
|
||||
def validate_path_safety(path_or_url, user):
|
||||
from toolbox import get_conf, default_user_name
|
||||
from toolbox import FriendlyException
|
||||
PATH_PRIVATE_UPLOAD, PATH_LOGGING = get_conf('PATH_PRIVATE_UPLOAD', 'PATH_LOGGING')
|
||||
sensitive_path = None
|
||||
path_or_url = os.path.relpath(path_or_url)
|
||||
if path_or_url.startswith(PATH_LOGGING): # 日志文件（按用户划分）
|
||||
sensitive_path = PATH_LOGGING
|
||||
elif path_or_url.startswith(PATH_PRIVATE_UPLOAD): # 用户的上传目录（按用户划分）
|
||||
sensitive_path = PATH_PRIVATE_UPLOAD
|
||||
elif path_or_url.startswith('tests'): # 一个常用的测试目录
|
||||
return True
|
||||
else:
|
||||
raise FriendlyException(f"输入文件的路径 ({path_or_url}) 存在，但位置非法。请将文件上传后再执行该任务。") # return False
|
||||
if sensitive_path:
|
||||
allowed_users = [user, 'autogen', default_user_name] # three user path that can be accessed
|
||||
for user_allowed in allowed_users:
|
||||
if f"{os.sep}".join(path_or_url.split(os.sep)[:2]) == os.path.join(sensitive_path, user_allowed):
|
||||
return True
|
||||
raise FriendlyException(f"输入文件的路径 ({path_or_url}) 存在，但属于其他用户。请将文件上传后再执行该任务。") # return False
|
||||
return True
|
||||
|
||||
def _authorize_user(path_or_url, request, gradio_app):
|
||||
from toolbox import get_conf, default_user_name
|
||||
PATH_PRIVATE_UPLOAD, PATH_LOGGING = get_conf('PATH_PRIVATE_UPLOAD', 'PATH_LOGGING')
|
||||
|
||||
22
toolbox.py
22
toolbox.py
@@ -1,3 +1,4 @@
|
||||
|
||||
import importlib
|
||||
import time
|
||||
import inspect
|
||||
@@ -10,6 +11,7 @@ import glob
|
||||
import logging
|
||||
import uuid
|
||||
from functools import wraps
|
||||
from textwrap import dedent
|
||||
from shared_utils.config_loader import get_conf
|
||||
from shared_utils.config_loader import set_conf
|
||||
from shared_utils.config_loader import set_multi_conf
|
||||
@@ -193,9 +195,20 @@ def trimmed_format_exc():
|
||||
replace_path = "."
|
||||
return str.replace(current_path, replace_path)
|
||||
|
||||
|
||||
def trimmed_format_exc_markdown():
|
||||
return '\n\n```\n' + trimmed_format_exc() + '```'
|
||||
|
||||
|
||||
class FriendlyException(Exception):
|
||||
def generate_error_html(self):
|
||||
return dedent(f"""
|
||||
<div class="center-div" style="color: crimson;text-align: center;">
|
||||
{"<br>".join(self.args)}
|
||||
</div>
|
||||
""")
|
||||
|
||||
|
||||
def CatchException(f):
|
||||
"""
|
||||
装饰器函数，捕捉函数f中的异常并封装到一个生成器中返回，并显示到聊天当中。
|
||||
@@ -206,13 +219,18 @@ def CatchException(f):
|
||||
chatbot_with_cookie:ChatBotWithCookies, history:list, *args, **kwargs):
|
||||
try:
|
||||
yield from f(main_input, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, *args, **kwargs)
|
||||
except FriendlyException as e:
|
||||
if len(chatbot_with_cookie) == 0:
|
||||
chatbot_with_cookie.clear()
|
||||
chatbot_with_cookie.append(["插件调度异常", None])
|
||||
chatbot_with_cookie[-1] = [chatbot_with_cookie[-1][0], e.generate_error_html()]
|
||||
yield from update_ui(chatbot=chatbot_with_cookie, history=history, msg=f'异常') # 刷新界面
|
||||
except Exception as e:
|
||||
from toolbox import get_conf
|
||||
tb_str = '```\n' + trimmed_format_exc() + '```'
|
||||
if len(chatbot_with_cookie) == 0:
|
||||
chatbot_with_cookie.clear()
|
||||
chatbot_with_cookie.append(["插件调度异常", "异常原因"])
|
||||
chatbot_with_cookie[-1] = (chatbot_with_cookie[-1][0], f"[Local Message] 插件调用出错: \n\n{tb_str} \n")
|
||||
chatbot_with_cookie[-1] = [chatbot_with_cookie[-1][0], f"[Local Message] 插件调用出错: \n\n{tb_str} \n"]
|
||||
yield from update_ui(chatbot=chatbot_with_cookie, history=history, msg=f'异常 {e}') # 刷新界面
|
||||
|
||||
return decorated
|
||||
|
||||
在新工单中引用
屏蔽一个用户