From a711db0b5b357b0ac886bd0531d1ffc06bc23403 Mon Sep 17 00:00:00 2001 From: qingxu fu <505030475@qq.com> Date: Wed, 25 Oct 2023 11:32:32 +0800 Subject: [PATCH 1/6] stashed commit --- config.py | 5 ++++ crazy_functions/crazy_utils.py | 9 +++--- crazy_functions/latex_fns/latex_actions.py | 22 ++++++++++++-- crazy_functions/latex_fns/latex_toolbox.py | 34 ++++++++++++++++++++++ tests/test_plugins.py | 4 ++- 5 files changed, 66 insertions(+), 8 deletions(-) diff --git a/config.py b/config.py index 387fab61..b76997fc 100644 --- a/config.py +++ b/config.py @@ -212,6 +212,11 @@ WHEN_TO_USE_PROXY = ["Download_LLM", "Download_Gradio_Theme", "Connect_Grobid", # 自定义按钮的最大数量限制 NUM_CUSTOM_BASIC_BTN = 4 + +# LATEX实验性功能 +LATEX_EXPERIMENTAL = False + + """ 在线大模型配置关联关系示意图 │ diff --git a/crazy_functions/crazy_utils.py b/crazy_functions/crazy_utils.py index 8533d088..8cc4042f 100644 --- a/crazy_functions/crazy_utils.py +++ b/crazy_functions/crazy_utils.py @@ -205,13 +205,12 @@ def request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( retry_op = retry_times_at_unknown_error exceeded_cnt = 0 mutable[index][2] = "执行中" + detect_timeout = lambda: len(mutable[index]) >= 2 and (time.time()-mutable[index][1]) > watch_dog_patience while True: # watchdog error - if len(mutable[index]) >= 2 and (time.time()-mutable[index][1]) > watch_dog_patience: - raise RuntimeError("检测到程序终止。") + if detect_timeout(): raise RuntimeError("检测到程序终止。") try: # 【第一种情况】：顺利完成 - # time.sleep(10); raise RuntimeError("测试") gpt_say = predict_no_ui_long_connection( inputs=inputs, llm_kwargs=llm_kwargs, history=history, sys_prompt=sys_prompt, observe_window=mutable[index], console_slience=True @@ -219,7 +218,7 @@ def request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( mutable[index][2] = "已成功" return gpt_say except ConnectionAbortedError as token_exceeded_error: - # 【第二种情况】：Token溢出， + # 【第二种情况】：Token溢出 if handle_token_exceed: exceeded_cnt += 1 # 【选择处理】 尝试计算比例，尽可能多地保留文本 @@ -240,6 +239,7 @@ def request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( return gpt_say # 放弃 except: # 【第三种情况】：其他错误 + if detect_timeout(): raise RuntimeError("检测到程序终止。") tb_str = '```\n' + trimmed_format_exc() + '```' print(tb_str) gpt_say += f"[Local Message] 警告，线程{index}在执行过程中遭遇问题, Traceback：\n\n{tb_str}\n\n" @@ -256,6 +256,7 @@ def request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( for i in range(wait): mutable[index][2] = f"{fail_info}等待重试 {wait-i}"; time.sleep(1) # 开始重试 + if detect_timeout(): raise RuntimeError("检测到程序终止。") mutable[index][2] = f"重试中 {retry_times_at_unknown_error-retry_op}/{retry_times_at_unknown_error}" continue # 返回重试 else: diff --git a/crazy_functions/latex_fns/latex_actions.py b/crazy_functions/latex_fns/latex_actions.py index 7e561df2..6232b197 100644 --- a/crazy_functions/latex_fns/latex_actions.py +++ b/crazy_functions/latex_fns/latex_actions.py @@ -1,9 +1,10 @@ from toolbox import update_ui, update_ui_lastest_msg, get_log_folder -from toolbox import zip_folder, objdump, objload, promote_file_to_downloadzone +from toolbox import get_conf, objdump, objload, promote_file_to_downloadzone from .latex_toolbox import PRESERVE, TRANSFORM from .latex_toolbox import set_forbidden_text, set_forbidden_text_begin_end, set_forbidden_text_careful_brace from .latex_toolbox import reverse_forbidden_text_careful_brace, reverse_forbidden_text, convert_to_linklist, post_process from .latex_toolbox import fix_content, find_main_tex_file, merge_tex_files, compile_latex_with_timeout +from .latex_toolbox import find_title_and_abs import os, shutil import re @@ -90,7 +91,15 @@ class LatexPaperSplit(): "项目Github地址 \\url{https://github.com/binary-husky/gpt_academic/}。" # 请您不要删除或修改这行警告，除非您是论文的原作者（如果您是论文原作者，欢迎加REAME中的QQ联系开发者） self.msg_declare = "为了防止大语言模型的意外谬误产生扩散影响，禁止移除或修改此警告。}}\\\\" + self.title = "unknown" + self.abstract = "unknown" + def read_title_and_abstract(self, txt): + title, abstract = find_title_and_abs(txt) + if title is not None: + self.title = title.replace('\n', ' ').replace('\\\\', ' ').replace(' ', '').replace(' ', '') + if abstract is not None: + self.abstract = abstract.replace('\n', ' ').replace('\\\\', ' ').replace(' ', '').replace(' ', '') def merge_result(self, arr, mode, msg, buggy_lines=[], buggy_line_surgery_n_lines=10): """ @@ -234,8 +243,8 @@ def Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin chatbot.append((f"Latex文件融合完成", f'[Local Message] 正在精细切分latex文件，这需要一段时间计算，文档越长耗时越长，请耐心等待。')) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 lps = LatexPaperSplit() + lps.read_title_and_abstract(merged_content) res = lps.split(merged_content, project_folder, opts) # 消耗时间的函数 - # <-------- 拆分过长的latex片段 ----------> pfg = LatexPaperFileGroup() for index, r in enumerate(res): @@ -256,12 +265,19 @@ def Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin else: # <-------- gpt 多线程请求 ----------> + LATEX_EXPERIMENTAL, = get_conf('LATEX_EXPERIMENTAL') + history_array = [[""] for _ in range(n_split)] + if LATEX_EXPERIMENTAL: + paper_meta = f"The paper you processing is `{lps.title}`, a part of the abstraction is `{lps.abstract}`" + paper_meta_max_len = 888 + history_array = [[ paper_meta[:paper_meta_max_len] + '...', "Understand, what should I do?"] for _ in range(n_split)] + gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( inputs_array=inputs_array, inputs_show_user_array=inputs_show_user_array, llm_kwargs=llm_kwargs, chatbot=chatbot, - history_array=[[""] for _ in range(n_split)], + history_array=history_array, sys_prompt_array=sys_prompt_array, # max_workers=5, # 并行任务数量限制, 最多同时执行5个, 其他的排队等待 scroller_max_len = 40 diff --git a/crazy_functions/latex_fns/latex_toolbox.py b/crazy_functions/latex_fns/latex_toolbox.py index 330cb657..b56825aa 100644 --- a/crazy_functions/latex_fns/latex_toolbox.py +++ b/crazy_functions/latex_fns/latex_toolbox.py @@ -318,6 +318,40 @@ def merge_tex_files_(project_foler, main_file, mode): main_file = main_file[:s.span()[0]] + c + main_file[s.span()[1]:] return main_file + +def find_title_and_abs(main_file): + + def extract_abstract_1(text): + pattern = r"\\abstract\{(.*?)\}" + match = re.search(pattern, text, re.DOTALL) + if match: + return match.group(1) + else: + return None + + def extract_abstract_2(text): + pattern = r"\\begin\{abstract\}(.*?)\\end\{abstract\}" + match = re.search(pattern, text, re.DOTALL) + if match: + return match.group(1) + else: + return None + + def extract_title(string): + pattern = r"\\title\{(.*?)\}" + match = re.search(pattern, string, re.DOTALL) + + if match: + return match.group(1) + else: + return None + + abstract = extract_abstract_1(main_file) + if abstract is None: + abstract = extract_abstract_2(main_file) + title = extract_title(main_file) + return title, abstract + def merge_tex_files(project_foler, main_file, mode): """ Merge Tex project recrusively diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 5998bc46..8470895d 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -11,7 +11,9 @@ if __name__ == "__main__": from tests.test_utils import plugin_test # plugin_test(plugin='crazy_functions.函数动态生成->函数动态生成', main_input='交换图像的蓝色通道和红色通道', advanced_arg={"file_path_arg": "./build/ants.jpg"}) - plugin_test(plugin='crazy_functions.Latex输出PDF结果->Latex翻译中文并重新编译PDF', main_input="2307.07522") + # plugin_test(plugin='crazy_functions.Latex输出PDF结果->Latex翻译中文并重新编译PDF', main_input="2307.07522") + + plugin_test(plugin='crazy_functions.Latex输出PDF结果->Latex翻译中文并重新编译PDF', main_input="G:/SEAFILE_LOCAL/50503047/我的资料库/学位/paperlatex/aaai/Fu_8368_with_appendix") # plugin_test(plugin='crazy_functions.虚空终端->虚空终端', main_input='修改api-key为sk-jhoejriotherjep') From 75a84d3cec1ce18b84638ea744ae55802ed1efac Mon Sep 17 00:00:00 2001 From: binary-husky <96192199+binary-husky@users.noreply.github.com> Date: Thu, 16 Nov 2023 17:18:07 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E6=B7=BB=E5=8A=A0python=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8e1e55b9..05ba22cd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ > **Note** > -> 2023.11.12: 紧急修复了endpoint异常的问题。 +> 2023.11.12: 某些依赖包尚不兼容python 3.12，推荐python 3.11。 > > 2023.11.7: 安装依赖时，请选择`requirements.txt`中**指定的版本**。 安装命令：`pip install -r requirements.txt`。本项目开源免费，近期发现有人蔑视开源协议并利用本项目违规圈钱，请提高警惕，谨防上当受骗。 @@ -108,7 +108,7 @@ cd gpt_academic 3. 安装依赖 ```sh -# （选择I: 如熟悉python, python>=3.9）备注：使用官方pip源或者阿里pip源, 临时换源方法：python -m pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ +# （选择I: 如熟悉python, python推荐版本 3.9 ~ 3.11）备注：使用官方pip源或者阿里pip源, 临时换源方法：python -m pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ python -m pip install -r requirements.txt # （选择II: 使用Anaconda）步骤也是类似的 (https://www.bilibili.com/video/BV1rc411W7Dr)： From 47fe06f79d28a65d06e92b5d062467b294ce1843 Mon Sep 17 00:00:00 2001 From: Mehdi Baneshi Date: Fri, 17 Nov 2023 05:45:47 +0330 Subject: [PATCH 3/6] Update README.English.md Change the reference section, add link for easy access to the resource --- docs/README.English.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/README.English.md b/docs/README.English.md index d0f3e4a1..f5632f35 100644 --- a/docs/README.English.md +++ b/docs/README.English.md @@ -326,25 +326,26 @@ You can change the theme by modifying the `THEME` option (config.py). 1. `master` branch: Main branch, stable version 2. `frontier` branch: Development branch, test version +*** + ### V: References and Learning -``` + The code references the designs of many other excellent projects, in no particular order: -# THU ChatGLM2-6B: -https://github.com/THUDM/ChatGLM2-6B +[THU ChatGLM2-6B](https://github.com/THUDM/ChatGLM2-6B) -# THU JittorLLMs: -https://github.com/Jittor/JittorLLMs -# ChatPaper: -https://github.com/kaixindelele/ChatPaper +[THU JittorLLMs](https://github.com/Jittor/JittorLLMs) -# Edge-GPT: -https://github.com/acheong08/EdgeGPT -# ChuanhuChatGPT: -https://github.com/GaiZhenbiao/ChuanhuChatGPT +[ChatPaper](https://github.com/kaixindelele/ChatPaper) + + +[Edge-GPT](https://github.com/acheong08/EdgeGPT) + + +[ChuanhuChatGPT](https://github.com/GaiZhenbiao/ChuanhuChatGPT) From 74f70305b7f1598be4a9149425cb4732d16e82c7 Mon Sep 17 00:00:00 2001 From: qingxu fu <505030475@qq.com> Date: Sun, 19 Nov 2023 22:03:36 +0800 Subject: [PATCH 4/6] introduce precommit --- .pre-commit-config.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..fd16ba2d --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,10 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files From 977f992e3a041eb8cff61e5fa527c251aefce97d Mon Sep 17 00:00:00 2001 From: qingxu fu <505030475@qq.com> Date: Mon, 20 Nov 2023 00:33:18 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=A4=9A=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E6=96=87=E4=BB=B6=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/对话历史存档.py | 26 ++++++--- crazy_functions/辅助功能.py | 22 ++++++-- toolbox.py | 98 ++++++++++++++++++++++++--------- 3 files changed, 106 insertions(+), 40 deletions(-) diff --git a/crazy_functions/对话历史存档.py b/crazy_functions/对话历史存档.py index f89faeda..3f603f9b 100644 --- a/crazy_functions/对话历史存档.py +++ b/crazy_functions/对话历史存档.py @@ -1,7 +1,8 @@ -from toolbox import CatchException, update_ui, promote_file_to_downloadzone, get_log_folder -from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive +from toolbox import CatchException, update_ui, promote_file_to_downloadzone, get_log_folder, get_user import re +f_prefix = 'GPT-Academic对话存档' + def write_chat_to_file(chatbot, history=None, file_name=None): """ 将对话记录history以Markdown格式写入文件中。如果没有指定文件名，则使用当前时间生成文件名。 @@ -9,8 +10,8 @@ def write_chat_to_file(chatbot, history=None, file_name=None): import os import time if file_name is None: - file_name = 'chatGPT对话历史' + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '.html' - fp = os.path.join(get_log_folder(), file_name) + file_name = f_prefix + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '.html' + fp = os.path.join(get_log_folder(get_user(chatbot), plugin_name='chat_history'), file_name) with open(fp, 'w', encoding='utf8') as f: from themes.theme import advanced_css f.write(f'对话历史') @@ -80,7 +81,7 @@ def 对话历史存档(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_ """ chatbot.append(("保存当前对话", - f"[Local Message] {write_chat_to_file(chatbot, history)}，您可以调用“载入对话历史存档”还原当下的对话。\n警告！被保存的对话历史可以被使用该系统的任何人查阅。")) + f"[Local Message] {write_chat_to_file(chatbot, history)}，您可以调用下拉菜单中的“载入对话历史存档”还原当下的对话。")) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 由于请求gpt需要一段时间，我们先及时地做一次界面更新 def hide_cwd(str): @@ -106,7 +107,12 @@ def 载入对话历史存档(txt, llm_kwargs, plugin_kwargs, chatbot, history, s if not success: if txt == "": txt = '空空如也的输入栏' import glob - local_history = "
".join(["`"+hide_cwd(f)+f" ({gen_file_preview(f)})"+"`" for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True)]) + local_history = "
".join([ + "`"+hide_cwd(f)+f" ({gen_file_preview(f)})"+"`" + for f in glob.glob( + f'{get_log_folder(get_user(chatbot), plugin_name="chat_history")}/**/{f_prefix}*.html', + recursive=True + )]) chatbot.append([f"正在查找对话历史文件（html格式）: {txt}", f"找不到任何html文件: {txt}。但本地存储了以下历史文件，您可以将任意一个文件路径粘贴到输入区，然后重试：
{local_history}"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return @@ -132,8 +138,12 @@ def 删除所有本地对话历史记录(txt, llm_kwargs, plugin_kwargs, chatbot """ import glob, os - local_history = "
".join(["`"+hide_cwd(f)+"`" for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True)]) - for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True): + local_history = "
".join([ + "`"+hide_cwd(f)+"`" + for f in glob.glob( + f'{get_log_folder(get_user(chatbot), plugin_name="chat_history")}/**/{f_prefix}*.html', recursive=True + )]) + for f in glob.glob(f'{get_log_folder(get_user(chatbot), plugin_name="chat_history")}/**/{f_prefix}*.html', recursive=True): os.remove(f) chatbot.append([f"删除所有历史对话文件", f"已删除
{local_history}"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/辅助功能.py b/crazy_functions/辅助功能.py index 16854e08..c5f87402 100644 --- a/crazy_functions/辅助功能.py +++ b/crazy_functions/辅助功能.py @@ -2,9 +2,12 @@ # @Time : 2023/4/19 # @Author : Spike # @Descr : -from toolbox import update_ui, get_conf +from toolbox import update_ui, get_conf, get_user from toolbox import CatchException +from toolbox import default_user_name from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive +import shutil +import os @CatchException @@ -33,10 +36,19 @@ def 清除缓存(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt chatbot.append(['清除本地缓存数据', '执行中. 删除数据']) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - import shutil, os - PATH_PRIVATE_UPLOAD, PATH_LOGGING = get_conf('PATH_PRIVATE_UPLOAD', 'PATH_LOGGING') - shutil.rmtree(PATH_LOGGING, ignore_errors=True) - shutil.rmtree(PATH_PRIVATE_UPLOAD, ignore_errors=True) + def _get_log_folder(user=default_user_name): + PATH_LOGGING = get_conf('PATH_LOGGING') + _dir = os.path.join(PATH_LOGGING, user) + if not os.path.exists(_dir): os.makedirs(_dir) + return _dir + + def _get_upload_folder(user=default_user_name): + PATH_PRIVATE_UPLOAD = get_conf('PATH_PRIVATE_UPLOAD') + _dir = os.path.join(PATH_PRIVATE_UPLOAD, user) + return _dir + + shutil.rmtree(_get_log_folder(get_user(chatbot)), ignore_errors=True) + shutil.rmtree(_get_upload_folder(get_user(chatbot)), ignore_errors=True) chatbot.append(['清除本地缓存数据', '执行完成']) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 \ No newline at end of file diff --git a/toolbox.py b/toolbox.py index b7b762d7..4376f050 100644 --- a/toolbox.py +++ b/toolbox.py @@ -11,7 +11,7 @@ import math from latex2mathml.converter import convert as tex2mathml from functools import wraps, lru_cache pj = os.path.join - +default_user_name = 'default_user' """ ======================================================================== 第一部分 @@ -61,11 +61,16 @@ def ArgsGeneralWrapper(f): txt_passon = txt if txt == "" and txt2 != "": txt_passon = txt2 # 引入一个有cookie的chatbot + if request.username is not None: + user_name = request.username + else: + user_name = default_user_name cookies.update({ 'top_p':top_p, 'api_key': cookies['api_key'], 'llm_model': llm_model, 'temperature':temperature, + 'user_name': user_name, }) llm_kwargs = { 'api_key': cookies['api_key'], @@ -537,40 +542,57 @@ def find_recent_files(directory): return recent_files + +def file_already_in_downloadzone(file, user_path): + parent_path = user_path + child_path = file + if os.path.commonpath([parent_path, child_path]) == parent_path: + return True + else: + return False + + def promote_file_to_downloadzone(file, rename_file=None, chatbot=None): # 将文件复制一份到下载区 import shutil - if rename_file is None: rename_file = f'{gen_time_str()}-{os.path.basename(file)}' - new_path = pj(get_log_folder(), rename_file) - # 如果已经存在，先删除 - if os.path.exists(new_path) and not os.path.samefile(new_path, file): os.remove(new_path) - # 把文件复制过去 - if not os.path.exists(new_path): shutil.copyfile(file, new_path) - # 将文件添加到chatbot cookie中，避免多用户干扰 + if chatbot is not None: + user_name = get_user(chatbot) + else: + user_name = default_user_name + + user_path = get_log_folder(user_name, plugin_name=None) + if file_already_in_downloadzone(file, user_path): + new_path = file + else: + if rename_file is None: rename_file = f'{gen_time_str()}-{os.path.basename(file)}' + new_path = pj(user_path, rename_file) + # 如果已经存在，先删除 + if os.path.exists(new_path) and not os.path.samefile(new_path, file): os.remove(new_path) + # 把文件复制过去 + if not os.path.exists(new_path): shutil.copyfile(file, new_path) + # 将文件添加到chatbot cookie中 if chatbot is not None: if 'files_to_promote' in chatbot._cookies: current = chatbot._cookies['files_to_promote'] else: current = [] chatbot._cookies.update({'files_to_promote': [new_path] + current}) return new_path + def disable_auto_promotion(chatbot): chatbot._cookies.update({'files_to_promote': []}) return -def is_the_upload_folder(string): - PATH_PRIVATE_UPLOAD = get_conf('PATH_PRIVATE_UPLOAD') - pattern = r'^PATH_PRIVATE_UPLOAD/[A-Za-z0-9_-]+/\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}$' - pattern = pattern.replace('PATH_PRIVATE_UPLOAD', PATH_PRIVATE_UPLOAD) - if re.match(pattern, string): return True - else: return False -def del_outdated_uploads(outdate_time_seconds): - PATH_PRIVATE_UPLOAD = get_conf('PATH_PRIVATE_UPLOAD') +def del_outdated_uploads(outdate_time_seconds, target_path_base=None): + if target_path_base is None: + user_upload_dir = get_conf('PATH_PRIVATE_UPLOAD') + else: + user_upload_dir = target_path_base current_time = time.time() one_hour_ago = current_time - outdate_time_seconds - # Get a list of all subdirectories in the PATH_PRIVATE_UPLOAD folder + # Get a list of all subdirectories in the user_upload_dir folder # Remove subdirectories that are older than one hour - for subdirectory in glob.glob(f'{PATH_PRIVATE_UPLOAD}/*/*'): + for subdirectory in glob.glob(f'{user_upload_dir}/*'): subdirectory_time = os.path.getmtime(subdirectory) if subdirectory_time < one_hour_ago: try: shutil.rmtree(subdirectory) @@ -583,17 +605,16 @@ def on_file_uploaded(request: gradio.Request, files, chatbot, txt, txt2, checkbo """ if len(files) == 0: return chatbot, txt - - # 移除过时的旧文件从而节省空间&保护隐私 - outdate_time_seconds = 60 - del_outdated_uploads(outdate_time_seconds) # 创建工作路径 - user_name = "default" if not request.username else request.username + user_name = default_user_name if not request.username else request.username time_tag = gen_time_str() - PATH_PRIVATE_UPLOAD = get_conf('PATH_PRIVATE_UPLOAD') - target_path_base = pj(PATH_PRIVATE_UPLOAD, user_name, time_tag) + target_path_base = get_upload_folder(user_name, tag=time_tag) os.makedirs(target_path_base, exist_ok=True) + + # 移除过时的旧文件从而节省空间&保护隐私 + outdate_time_seconds = 3600 # 一小时 + del_outdated_uploads(outdate_time_seconds, get_upload_folder(user_name)) # 逐个文件转移到目标路径 upload_msg = '' @@ -998,12 +1019,35 @@ def gen_time_str(): import time return time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) -def get_log_folder(user='default', plugin_name='shared'): +def get_log_folder(user=default_user_name, plugin_name='shared'): + if user is None: user = default_user_name PATH_LOGGING = get_conf('PATH_LOGGING') - _dir = pj(PATH_LOGGING, user, plugin_name) + if plugin_name is None: + _dir = pj(PATH_LOGGING, user) + else: + _dir = pj(PATH_LOGGING, user, plugin_name) if not os.path.exists(_dir): os.makedirs(_dir) return _dir +def get_upload_folder(user=default_user_name, tag=None): + PATH_PRIVATE_UPLOAD = get_conf('PATH_PRIVATE_UPLOAD') + if user is None: user = default_user_name + if tag is None or len(tag)==0: + target_path_base = pj(PATH_PRIVATE_UPLOAD, user) + else: + target_path_base = pj(PATH_PRIVATE_UPLOAD, user, tag) + return target_path_base + +def is_the_upload_folder(string): + PATH_PRIVATE_UPLOAD = get_conf('PATH_PRIVATE_UPLOAD') + pattern = r'^PATH_PRIVATE_UPLOAD[\\/][A-Za-z0-9_-]+[\\/]\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}$' + pattern = pattern.replace('PATH_PRIVATE_UPLOAD', PATH_PRIVATE_UPLOAD) + if re.match(pattern, string): return True + else: return False + +def get_user(chatbotwithcookies): + return chatbotwithcookies._cookies.get('user_name', default_user_name) + class ProxyNetworkActivate(): """ 这段代码定义了一个名为TempProxy的空上下文管理器, 用于给一小段代码上代理 From b1be05009bfc7b14478459d96a293252d0afcce3 Mon Sep 17 00:00:00 2001 From: qingxu fu <505030475@qq.com> Date: Mon, 20 Nov 2023 01:06:19 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E7=A7=BB=E9=99=A4=E5=86=97=E4=BD=99?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=EF=BC=8C=E4=BF=AE=E5=A4=8D=E5=A4=9A=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E5=AD=98=E6=A1=A3=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .pre-commit-config.yaml | 10 ---------- config.py | 5 ----- crazy_functions/latex_fns/latex_actions.py | 23 ++++++++++++---------- crazy_functions/latex_fns/latex_toolbox.py | 1 + toolbox.py | 15 ++++++++------ version | 2 +- 6 files changed, 24 insertions(+), 32 deletions(-) delete mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index fd16ba2d..00000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# See https://pre-commit.com for more information -# See https://pre-commit.com/hooks.html for more hooks -repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - - id: check-added-large-files diff --git a/config.py b/config.py index a51778b2..ea603c3b 100644 --- a/config.py +++ b/config.py @@ -235,11 +235,6 @@ BLOCK_INVALID_APIKEY = False # 自定义按钮的最大数量限制 NUM_CUSTOM_BASIC_BTN = 4 - -# LATEX实验性功能 -LATEX_EXPERIMENTAL = False - - """ 在线大模型配置关联关系示意图 │ diff --git a/crazy_functions/latex_fns/latex_actions.py b/crazy_functions/latex_fns/latex_actions.py index c51e9858..74e8757e 100644 --- a/crazy_functions/latex_fns/latex_actions.py +++ b/crazy_functions/latex_fns/latex_actions.py @@ -95,11 +95,14 @@ class LatexPaperSplit(): self.abstract = "unknown" def read_title_and_abstract(self, txt): - title, abstract = find_title_and_abs(txt) - if title is not None: - self.title = title.replace('\n', ' ').replace('\\\\', ' ').replace(' ', '').replace(' ', '') - if abstract is not None: - self.abstract = abstract.replace('\n', ' ').replace('\\\\', ' ').replace(' ', '').replace(' ', '') + try: + title, abstract = find_title_and_abs(txt) + if title is not None: + self.title = title.replace('\n', ' ').replace('\\\\', ' ').replace(' ', '').replace(' ', '') + if abstract is not None: + self.abstract = abstract.replace('\n', ' ').replace('\\\\', ' ').replace(' ', '').replace(' ', '') + except: + pass def merge_result(self, arr, mode, msg, buggy_lines=[], buggy_line_surgery_n_lines=10): """ @@ -265,12 +268,12 @@ def Latex精细分解与转化(file_manifest, project_folder, llm_kwargs, plugin else: # <-------- gpt 多线程请求 ----------> - LATEX_EXPERIMENTAL, = get_conf('LATEX_EXPERIMENTAL') history_array = [[""] for _ in range(n_split)] - if LATEX_EXPERIMENTAL: - paper_meta = f"The paper you processing is `{lps.title}`, a part of the abstraction is `{lps.abstract}`" - paper_meta_max_len = 888 - history_array = [[ paper_meta[:paper_meta_max_len] + '...', "Understand, what should I do?"] for _ in range(n_split)] + # LATEX_EXPERIMENTAL, = get_conf('LATEX_EXPERIMENTAL') + # if LATEX_EXPERIMENTAL: + # paper_meta = f"The paper you processing is `{lps.title}`, a part of the abstraction is `{lps.abstract}`" + # paper_meta_max_len = 888 + # history_array = [[ paper_meta[:paper_meta_max_len] + '...', "Understand, what should I do?"] for _ in range(n_split)] gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( inputs_array=inputs_array, diff --git a/crazy_functions/latex_fns/latex_toolbox.py b/crazy_functions/latex_fns/latex_toolbox.py index b56825aa..afa65bf9 100644 --- a/crazy_functions/latex_fns/latex_toolbox.py +++ b/crazy_functions/latex_fns/latex_toolbox.py @@ -352,6 +352,7 @@ def find_title_and_abs(main_file): title = extract_title(main_file) return title, abstract + def merge_tex_files(project_foler, main_file, mode): """ Merge Tex project recrusively diff --git a/toolbox.py b/toolbox.py index 4376f050..8747afdb 100644 --- a/toolbox.py +++ b/toolbox.py @@ -544,14 +544,16 @@ def find_recent_files(directory): def file_already_in_downloadzone(file, user_path): - parent_path = user_path - child_path = file - if os.path.commonpath([parent_path, child_path]) == parent_path: - return True - else: + try: + parent_path = os.path.abspath(user_path) + child_path = os.path.abspath(file) + if os.path.samefile(os.path.commonpath([parent_path, child_path]), parent_path): + return True + else: + return False + except: return False - def promote_file_to_downloadzone(file, rename_file=None, chatbot=None): # 将文件复制一份到下载区 import shutil @@ -564,6 +566,7 @@ def promote_file_to_downloadzone(file, rename_file=None, chatbot=None): if file_already_in_downloadzone(file, user_path): new_path = file else: + user_path = get_log_folder(user_name, plugin_name='downloadzone') if rename_file is None: rename_file = f'{gen_time_str()}-{os.path.basename(file)}' new_path = pj(user_path, rename_file) # 如果已经存在，先删除 diff --git a/version b/version index 81ad2fd9..cf14e82b 100644 --- a/version +++ b/version @@ -1,5 +1,5 @@ { "version": 3.60, "show_feature": true, - "new_feature": "11月12日紧急BUG修复 <-> AutoGen多智能体插件测试版 <-> 修复本地模型在Windows下的加载BUG <-> 支持文心一言v4和星火v3 <-> 支持GLM3和智谱的API <-> 解决本地模型并发BUG <-> 支持动态追加基础功能按钮 <-> 新汇报PDF汇总页面 <-> 重新编译Gradio优化使用体验" + "new_feature": "修复多个BUG <-> AutoGen多智能体插件测试版 <-> 修复本地模型在Windows下的加载BUG <-> 支持文心一言v4和星火v3 <-> 支持GLM3和智谱的API <-> 解决本地模型并发BUG <-> 支持动态追加基础功能按钮 <-> 新汇报PDF汇总页面 <-> 重新编译Gradio优化使用体验" }