@@ -405,4 +405,4 @@ def format_io(self, y):
# 输出部分
None if gpt_reply is None else markdown_convertion(gpt_reply),
)
- return y
+ return y
\ No newline at end of file
diff --git a/shared_utils/fastapi_server.py b/shared_utils/fastapi_server.py
index b30a5aaa..9159a07b 100644
--- a/shared_utils/fastapi_server.py
+++ b/shared_utils/fastapi_server.py
@@ -47,6 +47,28 @@ queue cocurrent effectiveness
import os, requests, threading, time
import uvicorn
+def validate_path_safety(path_or_url, user):
+ from toolbox import get_conf, default_user_name
+ from toolbox import FriendlyException
+ PATH_PRIVATE_UPLOAD, PATH_LOGGING = get_conf('PATH_PRIVATE_UPLOAD', 'PATH_LOGGING')
+ sensitive_path = None
+ path_or_url = os.path.relpath(path_or_url)
+ if path_or_url.startswith(PATH_LOGGING): # 日志文件（按用户划分）
+ sensitive_path = PATH_LOGGING
+ elif path_or_url.startswith(PATH_PRIVATE_UPLOAD): # 用户的上传目录（按用户划分）
+ sensitive_path = PATH_PRIVATE_UPLOAD
+ elif path_or_url.startswith('tests'): # 一个常用的测试目录
+ return True
+ else:
+ raise FriendlyException(f"输入文件的路径 ({path_or_url}) 存在，但位置非法。请将文件上传后再执行该任务。") # return False
+ if sensitive_path:
+ allowed_users = [user, 'autogen', default_user_name] # three user path that can be accessed
+ for user_allowed in allowed_users:
+ if f"{os.sep}".join(path_or_url.split(os.sep)[:2]) == os.path.join(sensitive_path, user_allowed):
+ return True
+ raise FriendlyException(f"输入文件的路径 ({path_or_url}) 存在，但属于其他用户。请将文件上传后再执行该任务。") # return False
+ return True
+
def _authorize_user(path_or_url, request, gradio_app):
from toolbox import get_conf, default_user_name
PATH_PRIVATE_UPLOAD, PATH_LOGGING = get_conf('PATH_PRIVATE_UPLOAD', 'PATH_LOGGING')
@@ -252,4 +274,4 @@ def start_app(app_block, CONCURRENT_COUNT, AUTHENTICATION, PORT, SSL_KEYFILE, SS
}
requests.get(f"{app_block.local_url}startup-events", verify=app_block.ssl_verify, proxies=forbid_proxies)
app_block.is_running = True
- app_block.block_thread()
+ app_block.block_thread()
\ No newline at end of file
diff --git a/toolbox.py b/toolbox.py
index d42e6075..0daeb59d 100644
--- a/toolbox.py
+++ b/toolbox.py
@@ -1,3 +1,4 @@
+
import importlib
import time
import inspect
@@ -10,6 +11,7 @@ import glob
import logging
import uuid
from functools import wraps
+from textwrap import dedent
from shared_utils.config_loader import get_conf
from shared_utils.config_loader import set_conf
from shared_utils.config_loader import set_multi_conf
@@ -193,9 +195,20 @@ def trimmed_format_exc():
replace_path = "."
return str.replace(current_path, replace_path)
+
def trimmed_format_exc_markdown():
return '\n\n```\n' + trimmed_format_exc() + '```'
+
+class FriendlyException(Exception):
+ def generate_error_html(self):
+ return dedent(f"""
+
+ {"
".join(self.args)}
+
+ """)
+
+
def CatchException(f):
"""
装饰器函数，捕捉函数f中的异常并封装到一个生成器中返回，并显示到聊天当中。
@@ -206,13 +219,18 @@ def CatchException(f):
chatbot_with_cookie:ChatBotWithCookies, history:list, *args, **kwargs):
try:
yield from f(main_input, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, *args, **kwargs)
+ except FriendlyException as e:
+ if len(chatbot_with_cookie) == 0:
+ chatbot_with_cookie.clear()
+ chatbot_with_cookie.append(["插件调度异常", None])
+ chatbot_with_cookie[-1] = [chatbot_with_cookie[-1][0], e.generate_error_html()]
+ yield from update_ui(chatbot=chatbot_with_cookie, history=history, msg=f'异常') # 刷新界面
except Exception as e:
- from toolbox import get_conf
tb_str = '```\n' + trimmed_format_exc() + '```'
if len(chatbot_with_cookie) == 0:
chatbot_with_cookie.clear()
chatbot_with_cookie.append(["插件调度异常", "异常原因"])
- chatbot_with_cookie[-1] = (chatbot_with_cookie[-1][0], f"[Local Message] 插件调用出错: \n\n{tb_str} \n")
+ chatbot_with_cookie[-1] = [chatbot_with_cookie[-1][0], f"[Local Message] 插件调用出错: \n\n{tb_str} \n"]
yield from update_ui(chatbot=chatbot_with_cookie, history=history, msg=f'异常 {e}') # 刷新界面
return decorated
From 15cc08505f7fbffb4ef1a9027f74e5e94b7a8947 Mon Sep 17 00:00:00 2001
From: binary-husky
Date: Wed, 19 Jun 2024 11:59:47 +0000
Subject: [PATCH 2/2] resolve safe pickle err
---
.gitignore | 2 +-
crazy_functions/latex_fns/latex_pickle_io.py | 12 ++-
tests/test_safe_pickle.py | 17 ++++
tests/test_save_chat_to_html.py | 102 +++++++++++++++++++
4 files changed, 128 insertions(+), 5 deletions(-)
create mode 100644 tests/test_safe_pickle.py
create mode 100644 tests/test_save_chat_to_html.py
diff --git a/.gitignore b/.gitignore
index 4fb8a7df..6d0e0cce 100644
--- a/.gitignore
+++ b/.gitignore
@@ -154,5 +154,5 @@ flagged
request_llms/ChatGLM-6b-onnx-u8s8
.pre-commit-config.yaml
themes/common.js.min.*.js
-test*
+test.html
objdump*
\ No newline at end of file
diff --git a/crazy_functions/latex_fns/latex_pickle_io.py b/crazy_functions/latex_fns/latex_pickle_io.py
index f08c78c1..451d735b 100644
--- a/crazy_functions/latex_fns/latex_pickle_io.py
+++ b/crazy_functions/latex_fns/latex_pickle_io.py
@@ -8,16 +8,20 @@ class SafeUnpickler(pickle.Unpickler):
# 定义允许的安全类
safe_classes = {
# 在这里添加其他安全的类
- 'latex_actions.LatexPaperFileGroup': LatexPaperFileGroup,
- 'latex_actions.LatexPaperSplit' : LatexPaperSplit,
+ 'LatexPaperFileGroup': LatexPaperFileGroup,
+ 'LatexPaperSplit' : LatexPaperSplit,
}
return safe_classes
def find_class(self, module, name):
# 只允许特定的类进行反序列化
self.safe_classes = self.get_safe_classes()
- if f'{module}.{name}' in self.safe_classes:
- return self.safe_classes[f'{module}.{name}']
+ match_class_name = None
+ for class_name in self.safe_classes.keys():
+ if (class_name in f'{module}.{name}'):
+ match_class_name = class_name
+ if match_class_name is not None:
+ return self.safe_classes[match_class_name]
# 如果尝试加载未授权的类，则抛出异常
raise pickle.UnpicklingError(f"Attempted to deserialize unauthorized class '{name}' from module '{module}'")
diff --git a/tests/test_safe_pickle.py b/tests/test_safe_pickle.py
new file mode 100644
index 00000000..01f69562
--- /dev/null
+++ b/tests/test_safe_pickle.py
@@ -0,0 +1,17 @@
+def validate_path():
+ import os, sys
+ os.path.dirname(__file__)
+ root_dir_assume = os.path.abspath(os.path.dirname(__file__) + "/..")
+ os.chdir(root_dir_assume)
+ sys.path.append(root_dir_assume)
+validate_path() # validate path so you can run from base directory
+
+from crazy_functions.latex_fns.latex_pickle_io import objdump, objload
+from crazy_functions.latex_fns.latex_actions import LatexPaperFileGroup, LatexPaperSplit
+pfg = LatexPaperFileGroup()
+pfg.get_token_num = None
+pfg.target = "target_elem"
+x = objdump(pfg)
+t = objload()
+
+print(t.target)
\ No newline at end of file
diff --git a/tests/test_save_chat_to_html.py b/tests/test_save_chat_to_html.py
new file mode 100644
index 00000000..8f69a266
--- /dev/null
+++ b/tests/test_save_chat_to_html.py
@@ -0,0 +1,102 @@
+def validate_path():
+ import os, sys
+ os.path.dirname(__file__)
+ root_dir_assume = os.path.abspath(os.path.dirname(__file__) + "/..")
+ os.chdir(root_dir_assume)
+ sys.path.append(root_dir_assume)
+validate_path() # validate path so you can run from base directory
+
+def write_chat_to_file(chatbot, history=None, file_name=None):
+ """
+ 将对话记录history以Markdown格式写入文件中。如果没有指定文件名，则使用当前时间生成文件名。
+ """
+ import os
+ import time
+ from themes.theme import advanced_css
+ # debug
+ import pickle
+ # def objdump(obj, file="objdump.tmp"):
+ # with open(file, "wb+") as f:
+ # pickle.dump(obj, f)
+ # return
+
+ def objload(file="objdump.tmp"):
+ import os
+ if not os.path.exists(file):
+ return
+ with open(file, "rb") as f:
+ return pickle.load(f)
+ # objdump((chatbot, history))
+ chatbot, history = objload()
+
+ with open("test.html", 'w', encoding='utf8') as f:
+ from textwrap import dedent
+ form = dedent("""
+ 对话存档
+
+
+
+
+ {CHAT_PREVIEW}
+
+
+
对话（原始数据）
+ {HISTORY_PREVIEW}
+
+
+
{QUESTION}
+
+
{ANSWER}
+