From baaacc5a7bbf9d61cbcbb19def7bd4a21679067f Mon Sep 17 00:00:00 2001 From: binary-husky <96192199+binary-husky@users.noreply.github.com> Date: Sat, 9 Sep 2023 19:11:21 +0800 Subject: [PATCH 01/29] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ef463bd1..04b2f630 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ Tip：不指定文件直接点击 `载入对话历史存档` 可以查看历史h 3. 虚空终端（从自然语言输入中，理解用户意图+自动调用其他插件） -- 步骤一：输入 “ 请调用插件翻译PDF论文，地址为https://storage.googleapis.com/deepmind-media/alphago/AlphaGoNaturePaper.pdf ” +- 步骤一：输入 “ 请调用插件翻译PDF论文，地址为https://openreview.net/pdf?id=rJl0r3R9KX ” - 步骤二：点击“虚空终端”
From 7c307d89647257f05823ff8e89f564e74a53ad79 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Sat, 9 Sep 2023 19:33:05 +0800 Subject: [PATCH 02/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=BA=90=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E8=A7=A3=E6=9E=90=E6=A8=A1=E5=9D=97=E4=B8=8E=E8=99=9A?= =?UTF-8?q?=E7=A9=BA=E7=BB=88=E7=AB=AF=E7=9A=84=E5=85=BC=E5=AE=B9=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/解析项目源代码.py | 10 ++++++---- docs/GithubAction+AllCapacity | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/crazy_functions/解析项目源代码.py b/crazy_functions/解析项目源代码.py index 30ae4440..926b629d 100644 --- a/crazy_functions/解析项目源代码.py +++ b/crazy_functions/解析项目源代码.py @@ -1,5 +1,5 @@ -from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import update_ui, promote_file_to_downloadzone +from toolbox import CatchException, report_execption, write_history_to_file from .crazy_utils import input_clipping def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt): @@ -43,7 +43,8 @@ def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, # 全部文件解析完成，结果写入文件，准备对工程源代码进行汇总分析 report_part_1 = copy.deepcopy(gpt_response_collection) history_to_return = report_part_1 - res = write_results_to_file(report_part_1) + res = write_history_to_file(report_part_1) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成？", "逐个文件分析已完成。" + res + "\n\n正在开始汇总。")) yield from update_ui(chatbot=chatbot, history=history_to_return) # 刷新界面 @@ -97,7 +98,8 @@ def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ############################## ################################## history_to_return.extend(report_part_2) - res = write_results_to_file(history_to_return) + res = write_history_to_file(history_to_return) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成了吗？", res)) yield from update_ui(chatbot=chatbot, history=history_to_return) # 刷新界面 diff --git a/docs/GithubAction+AllCapacity b/docs/GithubAction+AllCapacity index f6f05daa..82ddd087 100644 --- a/docs/GithubAction+AllCapacity +++ b/docs/GithubAction+AllCapacity @@ -1,7 +1,7 @@ # docker build -t gpt-academic-all-capacity -f docs/GithubAction+AllCapacity --network=host --build-arg http_proxy=http://localhost:10881 --build-arg https_proxy=http://localhost:10881 . # 从NVIDIA源，从而支持显卡（检查宿主的nvidia-smi中的cuda版本必须>=11.3） -FROM fuqingxu/11.3.1-runtime-ubuntu20.04-with-texlive:latest +FROM fuqingxu/11.3.1-runtime-ubuntu20.04-with-texlive-cache:latest # use python3 as the system default python WORKDIR /gpt From 2a5171507571e104f443bce05310301ebb931d81 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Sat, 9 Sep 2023 20:15:46 +0800 Subject: [PATCH 03/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8DDockerfile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/解析项目源代码.py | 5 +++-- docs/GithubAction+AllCapacity | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crazy_functions/解析项目源代码.py b/crazy_functions/解析项目源代码.py index 926b629d..debcb51b 100644 --- a/crazy_functions/解析项目源代码.py +++ b/crazy_functions/解析项目源代码.py @@ -1,4 +1,4 @@ -from toolbox import update_ui, promote_file_to_downloadzone +from toolbox import update_ui, promote_file_to_downloadzone, disable_auto_promotion from toolbox import CatchException, report_execption, write_history_to_file from .crazy_utils import input_clipping @@ -6,7 +6,8 @@ def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, import os, copy from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive - msg = '正常' + disable_auto_promotion(chatbot=chatbot) + summary_batch_isolation = True inputs_array = [] inputs_show_user_array = [] diff --git a/docs/GithubAction+AllCapacity b/docs/GithubAction+AllCapacity index 82ddd087..f6f05daa 100644 --- a/docs/GithubAction+AllCapacity +++ b/docs/GithubAction+AllCapacity @@ -1,7 +1,7 @@ # docker build -t gpt-academic-all-capacity -f docs/GithubAction+AllCapacity --network=host --build-arg http_proxy=http://localhost:10881 --build-arg https_proxy=http://localhost:10881 . # 从NVIDIA源，从而支持显卡（检查宿主的nvidia-smi中的cuda版本必须>=11.3） -FROM fuqingxu/11.3.1-runtime-ubuntu20.04-with-texlive-cache:latest +FROM fuqingxu/11.3.1-runtime-ubuntu20.04-with-texlive:latest # use python3 as the system default python WORKDIR /gpt From ad9807dd14eff0632ea4e2e07c377c07b81c364d Mon Sep 17 00:00:00 2001 From: binary-husky Date: Sat, 9 Sep 2023 20:32:44 +0800 Subject: [PATCH 04/29] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E8=99=9A=E7=A9=BA?= =?UTF-8?q?=E7=BB=88=E7=AB=AF=E7=9A=84=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/虚空终端.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crazy_functions/虚空终端.py b/crazy_functions/虚空终端.py index 2e1b5236..e01b274d 100644 --- a/crazy_functions/虚空终端.py +++ b/crazy_functions/虚空终端.py @@ -25,11 +25,12 @@ explain_msg = """ 1. 请用**自然语言**描述您需要做什么。例如： - 「请调用插件，为我翻译PDF论文，论文我刚刚放到上传区了」 - - 「请调用插件翻译PDF论文，地址为https://aaa/bbb/ccc.pdf」 + - 「请调用插件翻译PDF论文，地址为https://openreview.net/pdf?id=rJl0r3R9KX」 - 「把Arxiv论文翻译成中文PDF，arxiv论文的ID是1812.10695，记得用插件！」 - 「生成一张图片，图中鲜花怒放，绿草如茵，用插件实现」 - 「用插件翻译README，Github网址是https://github.com/facebookresearch/co-tracker」 - 「我不喜欢当前的界面颜色，修改配置，把主题THEME更换为THEME="High-Contrast"」 + - 「请调用插件，解析python源代码项目，代码我刚刚打包拖到上传区了」 - 「请问Transformer网络的结构是怎样的？」 2. 您可以打开插件下拉菜单以了解本项目的各种能力。 From c45fa88684ece949ee5bd19f0245d51a84eaaf71 Mon Sep 17 00:00:00 2001 From: qingxu fu <505030475@qq.com> Date: Sat, 9 Sep 2023 21:57:24 +0800 Subject: [PATCH 05/29] update translation matrix --- docs/translate_english.json | 46 ++++++++++++++++++++++++++++++++++++- docs/translate_std.json | 4 +++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/docs/translate_english.json b/docs/translate_english.json index 667956c8..a29c3664 100644 --- a/docs/translate_english.json +++ b/docs/translate_english.json @@ -2448,5 +2448,49 @@ "插件说明": "Plugin description", "├── CODE_HIGHLIGHT 代码高亮": "├── CODE_HIGHLIGHT Code highlighting", "记得用插件": "Remember to use the plugin", - "谨慎操作": "Handle with caution" + "谨慎操作": "Handle with caution", + "请检查PDF是否损坏": "#", + "执行成功了": "#", + "请在输入框内填写需求": "#", + "结果": "#", + "开始干正事": "#", + "次代码生成尝试": "#", + "代码生成结束": "#", + "Nougat解析论文失败": "#", + "受到google限制": "#", + "收尾": "#", + "结果是一个有效文件": "#", + "然后再次点击该插件": "#", + "用插件实现」": "#", + "文件路径": "#", + "仅供测试": "#", + "将csv文件转excel表格": "#", + "开始执行": "#", + "测试": "#", + "睡一会防止触发google反爬虫": "#", + "某段话的整个句子": "#", + "使用tex格式公式 测试2 给出柯西不等式": "#", + "找不到本地项目或无法处理": "#", + "交换图像的蓝色通道和红色通道": "#", + "第三步": "#", + "返回给定的url解析出的arxiv_id": "#", + "裁剪图像": "#", + "已经被记忆": "#", + "无法从bing获取信息！": "#", + "可能触发了google反爬虫机制": "#", + "检索文章的历史版本的题目": "#", + "请配置讯飞星火大模型的XFYUN_APPID": "#", + "执行失败了": "#", + "需要花费较长时间下载NOUGAT参数": "#", + "请检查": "#", + "写入": "#", + "下个句子中已经说完的部分": "#", + "精准翻译PDF文档": "#", + "解析python源代码项目": "#", + "首先在arxiv上搜索": "#", + "错误追踪": "#", + "结果是一个字符串": "#", + "由 test_on_sentence_end": "#", + "获取文章摘要": "#", + "受到bing限制": "#" } \ No newline at end of file diff --git a/docs/translate_std.json b/docs/translate_std.json index 63b6baa4..84690c2e 100644 --- a/docs/translate_std.json +++ b/docs/translate_std.json @@ -88,5 +88,7 @@ "辅助功能": "Accessibility", "虚空终端": "VoidTerminal", "解析PDF_基于GROBID": "ParsePDF_BasedOnGROBID", - "虚空终端主路由": "VoidTerminalMainRoute" + "虚空终端主路由": "VoidTerminalMainRoute", + "批量翻译PDF文档_NOUGAT": "BatchTranslatePDFDocuments_NOUGAT", + "解析PDF_基于NOUGAT": "ParsePDF_NOUGAT" } \ No newline at end of file From 28d777a96bfb92f86267797c5e2e6bc6ba0b7ae7 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Sun, 10 Sep 2023 16:52:35 +0800 Subject: [PATCH 06/29] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E6=8A=A5=E9=94=99?= =?UTF-8?q?=E6=B6=88=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/crazy_utils.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/crazy_functions/crazy_utils.py b/crazy_functions/crazy_utils.py index 5a314b37..15671725 100644 --- a/crazy_functions/crazy_utils.py +++ b/crazy_functions/crazy_utils.py @@ -469,14 +469,16 @@ def read_and_clean_pdf_text(fp): '- ', '') for t in text_areas['blocks'] if 'lines' in t] ############################## <第 2 步，获取正文主字体> ################################## - fsize_statiscs = {} - for span in meta_span: - if span[1] not in fsize_statiscs: fsize_statiscs[span[1]] = 0 - fsize_statiscs[span[1]] += span[2] - main_fsize = max(fsize_statiscs, key=fsize_statiscs.get) - if REMOVE_FOOT_NOTE: - give_up_fize_threshold = main_fsize * REMOVE_FOOT_FFSIZE_PERCENT - + try: + fsize_statiscs = {} + for span in meta_span: + if span[1] not in fsize_statiscs: fsize_statiscs[span[1]] = 0 + fsize_statiscs[span[1]] += span[2] + main_fsize = max(fsize_statiscs, key=fsize_statiscs.get) + if REMOVE_FOOT_NOTE: + give_up_fize_threshold = main_fsize * REMOVE_FOOT_FFSIZE_PERCENT + except: + raise RuntimeError(f'抱歉, 我们暂时无法解析此PDF文档: {fp}。') ############################## <第 3 步，切分和重新整合> ################################## mega_sec = [] sec = [] From 34784333dc29f9c8e8261d1a607c90ac1c3e7ed9 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Sun, 10 Sep 2023 17:22:35 +0800 Subject: [PATCH 07/29] =?UTF-8?q?=E8=9E=8D=E5=90=88PDF=E5=B7=A6=E5=8F=B3?= =?UTF-8?q?=E6=AF=94=E4=BE=8B=E8=B0=83=E6=95=B4=E5=88=B095%?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/latex_fns/latex_toolbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crazy_functions/latex_fns/latex_toolbox.py b/crazy_functions/latex_fns/latex_toolbox.py index 5adc7ea8..e39ac2db 100644 --- a/crazy_functions/latex_fns/latex_toolbox.py +++ b/crazy_functions/latex_fns/latex_toolbox.py @@ -423,7 +423,7 @@ def compile_latex_with_timeout(command, cwd, timeout=60): def merge_pdfs(pdf1_path, pdf2_path, output_path): import PyPDF2 - Percent = 0.8 + Percent = 0.95 # Open the first PDF file with open(pdf1_path, 'rb') as pdf1_file: pdf1_reader = PyPDF2.PdfFileReader(pdf1_file) From eb150987f0062e51bf003d3816ea6f8eb9c1cab3 Mon Sep 17 00:00:00 2001 From: qingxu fu <505030475@qq.com> Date: Mon, 11 Sep 2023 14:49:30 +0800 Subject: [PATCH 08/29] =?UTF-8?q?=E5=85=BC=E5=AE=B9=E4=B8=80=E4=B8=AAone-a?= =?UTF-8?q?pi=E6=B2=A1=E6=9C=89done=E6=95=B0=E6=8D=AE=E5=8C=85=E7=9A=84?= =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9Bug=E6=83=85=E5=BD=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request_llm/bridge_chatgpt.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/request_llm/bridge_chatgpt.py b/request_llm/bridge_chatgpt.py index 929a7546..19d10022 100644 --- a/request_llm/bridge_chatgpt.py +++ b/request_llm/bridge_chatgpt.py @@ -179,8 +179,13 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp # 非OpenAI官方接口的出现这样的报错，OpenAI和API2D不会走这里 chunk_decoded = chunk.decode() error_msg = chunk_decoded + # 首先排除一个one-api没有done数据包的第三方Bug情形 + if len(gpt_replying_buffer.strip()) > 0 and len(error_msg) == 0: + yield from update_ui(chatbot=chatbot, history=history, msg="检测到有缺陷的非OpenAI官方接口，建议选择更稳定的接口。") + break + # 其他情况，直接返回报错 chatbot, history = handle_error(inputs, llm_kwargs, chatbot, history, chunk_decoded, error_msg) - yield from update_ui(chatbot=chatbot, history=history, msg="非Openai官方接口返回了错误:" + chunk.decode()) # 刷新界面 + yield from update_ui(chatbot=chatbot, history=history, msg="非OpenAI官方接口返回了错误:" + chunk.decode()) # 刷新界面 return chunk_decoded = chunk.decode() From d495b73456eb504d4cf995ee910ce387d28e4600 Mon Sep 17 00:00:00 2001 From: qingxu fu <505030475@qq.com> Date: Mon, 11 Sep 2023 22:55:32 +0800 Subject: [PATCH 09/29] =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=9B=B4=E5=A4=9AUI?= =?UTF-8?q?=E7=9A=AE=E8=82=A4=E5=A4=96=E8=A7=82=EF=BC=8C=E5=8A=A0=E5=85=A5?= =?UTF-8?q?=E6=9A=97=E8=89=B2=E4=BA=AE=E8=89=B2=E5=88=87=E6=8D=A2=E9=94=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 3 ++- main.py | 9 +++++++++ themes/gradios.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ themes/theme.py | 3 +++ version | 4 ++-- 5 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 themes/gradios.py diff --git a/config.py b/config.py index 1e91e39c..2d57f09b 100644 --- a/config.py +++ b/config.py @@ -43,7 +43,8 @@ API_URL_REDIRECT = {} DEFAULT_WORKER_NUM = 3 -# 色彩主题，可选 ["Default", "Chuanhu-Small-and-Beautiful", "High-Contrast"] +# 色彩主题, 可选 ["Default", "Chuanhu-Small-and-Beautiful", "High-Contrast"] +# 更多主题, 请查阅Gradio主题商店: https://huggingface.co/spaces/gradio/theme-gallery 可选 ["Gstaff/Xkcd", "NoCrypt/Miku", ...] THEME = "Default" diff --git a/main.py b/main.py index d4752535..628d30b9 100644 --- a/main.py +++ b/main.py @@ -123,6 +123,15 @@ def main(): max_length_sl = gr.Slider(minimum=256, maximum=8192, value=4096, step=1, interactive=True, label="Local LLM MaxLength",) checkboxes = gr.CheckboxGroup(["基础功能区", "函数插件区", "底部输入区", "输入清除键", "插件参数区"], value=["基础功能区", "函数插件区"], label="显示/隐藏功能区") md_dropdown = gr.Dropdown(AVAIL_LLM_MODELS, value=LLM_MODEL, label="更换LLM模型/请求源").style(container=False) + dark_mode_btn = gr.Button("Toggle Dark Mode ☀", variant="secondary").style(size="sm") + dark_mode_btn.click(None, None, None, _js="""() => { + if (document.querySelectorAll('.dark').length) { + document.querySelectorAll('.dark').forEach(el => el.classList.remove('dark')); + } else { + document.querySelector('body').classList.add('dark'); + } + }""", + ) gr.Markdown(description) with gr.Accordion("备选输入区", open=True, visible=False, elem_id="input-panel2") as area_input_secondary: with gr.Row(): diff --git a/themes/gradios.py b/themes/gradios.py new file mode 100644 index 00000000..8b661a56 --- /dev/null +++ b/themes/gradios.py @@ -0,0 +1,46 @@ +import gradio as gr +import logging +from toolbox import get_conf, ProxyNetworkActivate +CODE_HIGHLIGHT, ADD_WAIFU, LAYOUT = get_conf('CODE_HIGHLIGHT', 'ADD_WAIFU', 'LAYOUT') + +def adjust_theme(): + + try: + set_theme = gr.themes.ThemeClass() + with ProxyNetworkActivate(): + logging.info('正在下载Gradio主题，请稍等。') + THEME, = get_conf('THEME') + if THEME.startswith('Huggingface-'): THEME = THEME.lstrip('Huggingface-') + if THEME.startswith('huggingface-'): THEME = THEME.lstrip('huggingface-') + set_theme = set_theme.from_hub(THEME.lower()) + + if LAYOUT=="TOP-DOWN": + js = "" + else: + with open('themes/common.js', 'r', encoding='utf8') as f: + js = f"" + + # 添加一个萌萌的看板娘 + if ADD_WAIFU: + js += """ + + + + """ + gradio_original_template_fn = gr.routes.templates.TemplateResponse + def gradio_new_template_fn(*args, **kwargs): + res = gradio_original_template_fn(*args, **kwargs) + res.body = res.body.replace(b'', f'{js}'.encode("utf8")) + res.init_headers() + return res + gr.routes.templates.TemplateResponse = gradio_new_template_fn # override gradio template + except Exception as e: + set_theme = None + from toolbox import trimmed_format_exc + logging.error('gradio版本较旧, 不能自定义字体和颜色:', trimmed_format_exc()) + return set_theme + +# with open("themes/default.css", "r", encoding="utf-8") as f: +# advanced_css = f.read() +with open("themes/common.css", "r", encoding="utf-8") as f: + advanced_css = f.read() diff --git a/themes/theme.py b/themes/theme.py index 5cba541c..dbb8f1e2 100644 --- a/themes/theme.py +++ b/themes/theme.py @@ -8,6 +8,9 @@ if THEME == 'Chuanhu-Small-and-Beautiful': elif THEME == 'High-Contrast': from .contrast import adjust_theme, advanced_css theme_declaration = "" +elif '/' in THEME: + from .gradios import adjust_theme, advanced_css + theme_declaration = "" else: from .default import adjust_theme, advanced_css theme_declaration = "" diff --git a/version b/version index 369a8008..4f234c6f 100644 --- a/version +++ b/version @@ -1,5 +1,5 @@ { - "version": 3.50, + "version": 3.51, "show_feature": true, - "new_feature": "支持插件分类！ <-> 支持用户使用自然语言调度各个插件(虚空终端) ！ <-> 改进UI，设计新主题 <-> 支持借助GROBID实现PDF高精度翻译 <-> 接入百度千帆平台和文心一言 <-> 接入阿里通义千问、讯飞星火、上海AI-Lab书生 <-> 优化一键升级 <-> 提高arxiv翻译速度和成功率" + "new_feature": "支持插件分类和更多UI皮肤外观 <-> 支持用户使用自然语言调度各个插件(虚空终端) ！ <-> 改进UI，设计新主题 <-> 支持借助GROBID实现PDF高精度翻译 <-> 接入百度千帆平台和文心一言 <-> 接入阿里通义千问、讯飞星火、上海AI-Lab书生 <-> 优化一键升级 <-> 提高arxiv翻译速度和成功率" } From 4b5f13bff26c2412078a4af550dc265fa37201df Mon Sep 17 00:00:00 2001 From: qingxu fu <505030475@qq.com> Date: Tue, 12 Sep 2023 11:35:31 +0800 Subject: [PATCH 10/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=9F=A5=E8=AF=86?= =?UTF-8?q?=E5=BA=93=E7=9A=84=E4=BE=9D=E8=B5=96=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functional.py | 2 +- crazy_functions/Langchain知识库.py | 37 +++++++++++++++--------------- crazy_functions/crazy_utils.py | 9 +++++--- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/crazy_functional.py b/crazy_functional.py index c6578554..b0098833 100644 --- a/crazy_functional.py +++ b/crazy_functional.py @@ -415,7 +415,7 @@ def get_crazy_functions(): try: from crazy_functions.Langchain知识库 import 读取知识库作答 function_plugins.update({ - "知识库问答": { + "知识库问答（请构建知识库）": { "Group": "对话", "Color": "stop", "AsButton": False, diff --git a/crazy_functions/Langchain知识库.py b/crazy_functions/Langchain知识库.py index 12735dfd..741a3d06 100644 --- a/crazy_functions/Langchain知识库.py +++ b/crazy_functions/Langchain知识库.py @@ -1,4 +1,4 @@ -from toolbox import CatchException, update_ui, ProxyNetworkActivate +from toolbox import CatchException, update_ui, ProxyNetworkActivate, update_ui_lastest_msg from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive, get_files_from_everything @@ -15,7 +15,12 @@ def 知识库问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_pro web_port 当前软件运行的端口号 """ history = [] # 清空历史，以免输入溢出 - chatbot.append(("这是什么功能？", "[Local Message] 从一批文件(txt, md, tex)中读取数据构建知识库, 然后进行问答。")) + + # < --------------------读取参数--------------- > + if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") + kai_id = plugin_kwargs.get("advanced_arg", 'default') + + chatbot.append((f"向`{kai_id}`知识库中添加文件。", "[Local Message] 从一批文件(txt, md, tex)中读取数据构建知识库, 然后进行问答。")) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # resolve deps @@ -24,17 +29,12 @@ def 知识库问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_pro from langchain.embeddings.huggingface import HuggingFaceEmbeddings from .crazy_utils import knowledge_archive_interface except Exception as e: - chatbot.append( - ["依赖不足", - "导入依赖失败。正在尝试自动安装，请查看终端的输出或耐心等待..."] - ) + chatbot.append(["依赖不足", "导入依赖失败。正在尝试自动安装，请查看终端的输出或耐心等待..."]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 from .crazy_utils import try_install_deps - try_install_deps(['zh_langchain==0.2.1', 'pypinyin']) - - # < --------------------读取参数--------------- > - if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") - kai_id = plugin_kwargs.get("advanced_arg", 'default') + try_install_deps(['zh_langchain==0.2.1', 'pypinyin'], reload_m=['pypinyin', 'zh_langchain']) + yield from update_ui_lastest_msg("安装完成，您可以再次重试。", chatbot, history) + return # < --------------------读取文件--------------- > file_manifest = [] @@ -84,19 +84,18 @@ def 读取知识库作答(txt, llm_kwargs, plugin_kwargs, chatbot, history, syst chatbot.append(["依赖不足", "导入依赖失败。正在尝试自动安装，请查看终端的输出或耐心等待..."]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 from .crazy_utils import try_install_deps - try_install_deps(['zh_langchain==0.2.1']) + try_install_deps(['zh_langchain==0.2.1', 'pypinyin'], reload_m=['pypinyin', 'zh_langchain']) + yield from update_ui_lastest_msg("安装完成，您可以再次重试。", chatbot, history) + return # < ------------------- --------------- > kai = knowledge_archive_interface() - if 'langchain_plugin_embedding' in chatbot._cookies: - resp, prompt = kai.answer_with_archive_by_id(txt, chatbot._cookies['langchain_plugin_embedding']) - else: - if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") - kai_id = plugin_kwargs.get("advanced_arg", 'default') - resp, prompt = kai.answer_with_archive_by_id(txt, kai_id) + if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg") + kai_id = plugin_kwargs.get("advanced_arg", 'default') + resp, prompt = kai.answer_with_archive_by_id(txt, kai_id) - chatbot.append((txt, '[Local Message] ' + prompt)) + chatbot.append((txt, f'[知识库 {kai_id}] ' + prompt)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 由于请求gpt需要一段时间，我们先及时地做一次界面更新 gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive( inputs=prompt, inputs_show_user=txt, diff --git a/crazy_functions/crazy_utils.py b/crazy_functions/crazy_utils.py index 15671725..9c77b8a7 100644 --- a/crazy_functions/crazy_utils.py +++ b/crazy_functions/crazy_utils.py @@ -706,11 +706,14 @@ class knowledge_archive_interface(): self.threadLock.release() return resp, prompt -def try_install_deps(deps): +def try_install_deps(deps, reload_m=[]): + import subprocess, sys, importlib for dep in deps: - import subprocess, sys subprocess.check_call([sys.executable, '-m', 'pip', 'install', '--user', dep]) - + import site + importlib.reload(site) + for m in reload_m: + importlib.reload(__import__(m)) class construct_html(): def __init__(self) -> None: From 7e73a255f423735ebffde209c07ed36ce45d8477 Mon Sep 17 00:00:00 2001 From: qingxu fu <505030475@qq.com> Date: Tue, 12 Sep 2023 11:47:34 +0800 Subject: [PATCH 11/29] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=9F=A5=E8=AF=86?= =?UTF-8?q?=E5=BA=93=E6=8F=92=E4=BB=B6=E7=9A=84=E6=8F=90=E7=A4=BA=E4=BF=A1?= =?UTF-8?q?=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functional.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crazy_functional.py b/crazy_functional.py index b0098833..c5ccda7c 100644 --- a/crazy_functional.py +++ b/crazy_functional.py @@ -400,12 +400,12 @@ def get_crazy_functions(): try: from crazy_functions.Langchain知识库 import 知识库问答 function_plugins.update({ - "构建知识库（请先上传文件素材）": { + "构建知识库（先上传文件素材,再运行此插件）": { "Group": "对话", "Color": "stop", "AsButton": False, "AdvancedArgs": True, - "ArgsReminder": "待注入的知识库名称id, 默认为default", + "ArgsReminder": "此处待注入的知识库名称id, 默认为default。文件进入知识库后可长期保存。可以通过再次调用本插件的方式，向知识库追加更多文档。", "Function": HotReload(知识库问答) } }) @@ -415,12 +415,12 @@ def get_crazy_functions(): try: from crazy_functions.Langchain知识库 import 读取知识库作答 function_plugins.update({ - "知识库问答（请构建知识库）": { + "知识库问答（构建知识库后,再运行此插件）": { "Group": "对话", "Color": "stop", "AsButton": False, "AdvancedArgs": True, - "ArgsReminder": "待提取的知识库名称id, 默认为default, 您需要首先调用构建知识库", + "ArgsReminder": "待提取的知识库名称id, 默认为default, 您需要构建知识库后再运行此插件。", "Function": HotReload(读取知识库作答) } }) From 99a1cd6f9f0d084b2b82068334e8daadd4d5634a Mon Sep 17 00:00:00 2001 From: qingxu fu <505030475@qq.com> Date: Tue, 12 Sep 2023 12:20:05 +0800 Subject: [PATCH 12/29] =?UTF-8?q?=E6=B7=BB=E5=8A=A0pypinyin=E4=BE=9D?= =?UTF-8?q?=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/GithubAction+AllCapacity | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/GithubAction+AllCapacity b/docs/GithubAction+AllCapacity index f6f05daa..2ed9f4c9 100644 --- a/docs/GithubAction+AllCapacity +++ b/docs/GithubAction+AllCapacity @@ -12,7 +12,7 @@ RUN python3 -m pip install torch --extra-index-url https://download.pytorch.org/ RUN python3 -m pip install openai numpy arxiv rich RUN python3 -m pip install colorama Markdown pygments pymupdf RUN python3 -m pip install python-docx moviepy pdfminer -RUN python3 -m pip install zh_langchain==0.2.1 +RUN python3 -m pip install zh_langchain==0.2.1 pypinyin RUN python3 -m pip install nougat-ocr RUN python3 -m pip install rarfile py7zr RUN python3 -m pip install aliyun-python-sdk-core==2.13.3 pyOpenSSL scipy git+https://github.com/aliyun/alibabacloud-nls-python-sdk.git From c4c64659276a6fe185977e34537f041d5383e1fb Mon Sep 17 00:00:00 2001 From: binary-husky Date: Tue, 12 Sep 2023 18:57:50 +0800 Subject: [PATCH 13/29] =?UTF-8?q?=E8=A7=A3=E5=86=B3issues=20#1097?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/latex_fns/latex_actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crazy_functions/latex_fns/latex_actions.py b/crazy_functions/latex_fns/latex_actions.py index c59bc31d..04efc94a 100644 --- a/crazy_functions/latex_fns/latex_actions.py +++ b/crazy_functions/latex_fns/latex_actions.py @@ -363,7 +363,7 @@ def 编译Latex(chatbot, history, main_file_original, main_file_modified, work_f if mode!='translate_zh': yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 使用latexdiff生成论文转化前后对比 ...', chatbot, history) # 刷新Gradio前端界面 print( f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex') - ok = compile_latex_with_timeout(f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex') + ok = compile_latex_with_timeout(f'latexdiff --encoding=utf8 --append-safecmd=subfile {work_folder_original}/{main_file_original}.tex {work_folder_modified}/{main_file_modified}.tex --flatten > {work_folder}/merge_diff.tex', os.getcwd()) yield from update_ui_lastest_msg(f'尝试第 {n_fix}/{max_try} 次编译, 正在编译对比PDF ...', chatbot, history) # 刷新Gradio前端界面 ok = compile_latex_with_timeout(f'pdflatex -interaction=batchmode -file-line-error merge_diff.tex', work_folder) From fb5467b85b6a93520749a2fd3ed561ac6e07020b Mon Sep 17 00:00:00 2001 From: binary-husky Date: Tue, 12 Sep 2023 19:13:36 +0800 Subject: [PATCH 14/29] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=8F=92=E4=BB=B6?= =?UTF-8?q?=E7=B3=BB=E7=BB=9F=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/Latex全文润色.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crazy_functions/Latex全文润色.py b/crazy_functions/Latex全文润色.py index 9e1d4b66..9aaff6c0 100644 --- a/crazy_functions/Latex全文润色.py +++ b/crazy_functions/Latex全文润色.py @@ -137,7 +137,7 @@ def Latex英文润色(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_p # 基本信息：功能、贡献者 chatbot.append([ "函数插件功能？", - "对整个Latex项目进行润色。函数插件贡献者: Binary-Husky"]) + "对整个Latex项目进行润色。函数插件贡献者: Binary-Husky。（注意，此插件不调用Latex，如果有Latex环境，请使用“Latex英文纠错+高亮”插件）"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 尝试导入依赖，如果缺少依赖，则给出安装建议 From 14de282302ebfbc024f34e6bedfcc2a9a2047efc Mon Sep 17 00:00:00 2001 From: binary-husky Date: Wed, 13 Sep 2023 23:21:00 +0800 Subject: [PATCH 15/29] =?UTF-8?q?=E7=BB=99nougat=E5=8A=A0=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E9=94=81=20=E5=90=88=E5=B9=B6=E5=86=97=E4=BD=99?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/crazy_utils.py | 68 ++++++++++++++++----- crazy_functions/批量翻译PDF文档_NOUGAT.py | 74 +---------------------- crazy_functions/批量翻译PDF文档_多线程.py | 48 +-------------- 3 files changed, 57 insertions(+), 133 deletions(-) diff --git a/crazy_functions/crazy_utils.py b/crazy_functions/crazy_utils.py index 9c77b8a7..6cf471e5 100644 --- a/crazy_functions/crazy_utils.py +++ b/crazy_functions/crazy_utils.py @@ -1,5 +1,6 @@ -from toolbox import update_ui, get_conf, trimmed_format_exc +from toolbox import update_ui, get_conf, trimmed_format_exc, get_log_folder import threading +import os def input_clipping(inputs, history, max_token_limit): import numpy as np @@ -705,6 +706,40 @@ class knowledge_archive_interface(): ) self.threadLock.release() return resp, prompt + +@Singleton +class nougat_interface(): + def __init__(self): + self.threadLock = threading.Lock() + + def nougat_with_timeout(self, command, cwd, timeout=3600): + import subprocess + process = subprocess.Popen(command, shell=True, cwd=cwd) + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + stdout, stderr = process.communicate() + print("Process timed out!") + return False + return True + + + def NOUGAT_parse_pdf(self, fp): + self.threadLock.acquire() + import glob, threading, os + from toolbox import get_log_folder, gen_time_str + dst = os.path.join(get_log_folder(plugin_name='nougat'), gen_time_str()) + os.makedirs(dst) + self.nougat_with_timeout(f'nougat --out "{os.path.abspath(dst)}" "{os.path.abspath(fp)}"', os.getcwd()) + res = glob.glob(os.path.join(dst,'*.mmd')) + if len(res) == 0: + raise RuntimeError("Nougat解析论文失败。") + self.threadLock.release() + return res[0] + + + def try_install_deps(deps, reload_m=[]): import subprocess, sys, importlib @@ -715,42 +750,43 @@ def try_install_deps(deps, reload_m=[]): for m in reload_m: importlib.reload(__import__(m)) -class construct_html(): - def __init__(self) -> None: - self.css = """ + +HTML_CSS = """ .row { display: flex; flex-wrap: wrap; } - .column { flex: 1; padding: 10px; } - .table-header { font-weight: bold; border-bottom: 1px solid black; } - .table-row { border-bottom: 1px solid lightgray; } - .table-cell { padding: 5px; } - """ - self.html_string = f'翻译结果' +""" - - def add_row(self, a, b): - tmp = """ +TABLE_CSS = """
REPLACE_A
REPLACE_B
- """ +""" + +class construct_html(): + def __init__(self) -> None: + self.css = HTML_CSS + self.html_string = f'翻译结果' + + + def add_row(self, a, b): + tmp = TABLE_CSS from toolbox import markdown_convertion tmp = tmp.replace('REPLACE_A', markdown_convertion(a)) tmp = tmp.replace('REPLACE_B', markdown_convertion(b)) @@ -758,6 +794,6 @@ class construct_html(): def save_file(self, file_name): - with open(f'./gpt_log/{file_name}', 'w', encoding='utf8') as f: + with open(os.path.join(get_log_folder(), file_name), 'w', encoding='utf8') as f: f.write(self.html_string.encode('utf-8', 'ignore').decode()) - + return os.path.join(get_log_folder(), file_name) diff --git a/crazy_functions/批量翻译PDF文档_NOUGAT.py b/crazy_functions/批量翻译PDF文档_NOUGAT.py index ed151211..0e2fb81d 100644 --- a/crazy_functions/批量翻译PDF文档_NOUGAT.py +++ b/crazy_functions/批量翻译PDF文档_NOUGAT.py @@ -86,31 +86,8 @@ def 批量翻译PDF文档(txt, llm_kwargs, plugin_kwargs, chatbot, history, syst # 开始正式执行任务 yield from 解析PDF_基于NOUGAT(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt) - -def nougat_with_timeout(command, cwd, timeout=3600): - import subprocess - process = subprocess.Popen(command, shell=True, cwd=cwd) - try: - stdout, stderr = process.communicate(timeout=timeout) - except subprocess.TimeoutExpired: - process.kill() - stdout, stderr = process.communicate() - print("Process timed out!") - return False - return True -def NOUGAT_parse_pdf(fp): - import glob - from toolbox import get_log_folder, gen_time_str - dst = os.path.join(get_log_folder(plugin_name='nougat'), gen_time_str()) - os.makedirs(dst) - nougat_with_timeout(f'nougat --out "{os.path.abspath(dst)}" "{os.path.abspath(fp)}"', os.getcwd()) - res = glob.glob(os.path.join(dst,'*.mmd')) - if len(res) == 0: - raise RuntimeError("Nougat解析论文失败。") - return res[0] - def 解析PDF_基于NOUGAT(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt): import copy @@ -119,9 +96,11 @@ def 解析PDF_基于NOUGAT(file_manifest, project_folder, llm_kwargs, plugin_kwa generated_conclusion_files = [] generated_html_files = [] DST_LANG = "中文" + from crazy_functions.crazy_utils import nougat_interface, construct_html + nougat_handle = nougat_interface() for index, fp in enumerate(file_manifest): chatbot.append(["当前进度：", f"正在解析论文，请稍候。（第一次运行时，需要花费较长时间下载NOUGAT参数）"]); yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - fpp = NOUGAT_parse_pdf(fp) + fpp = nougat_handle.NOUGAT_parse_pdf(fp) with open(fpp, 'r', encoding='utf8') as f: article_content = f.readlines() @@ -222,50 +201,3 @@ def 解析PDF_基于NOUGAT(file_manifest, project_folder, llm_kwargs, plugin_kwa yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - -class construct_html(): - def __init__(self) -> None: - self.css = """ -.row { - display: flex; - flex-wrap: wrap; -} - -.column { - flex: 1; - padding: 10px; -} - -.table-header { - font-weight: bold; - border-bottom: 1px solid black; -} - -.table-row { - border-bottom: 1px solid lightgray; -} - -.table-cell { - padding: 5px; -} - """ - self.html_string = f'翻译结果' - - - def add_row(self, a, b): - tmp = """ -
-
REPLACE_A
-
REPLACE_B
-
- """ - from toolbox import markdown_convertion - tmp = tmp.replace('REPLACE_A', markdown_convertion(a)) - tmp = tmp.replace('REPLACE_B', markdown_convertion(b)) - self.html_string += tmp - - - def save_file(self, file_name): - with open(os.path.join(get_log_folder(), file_name), 'w', encoding='utf8') as f: - f.write(self.html_string.encode('utf-8', 'ignore').decode()) - return os.path.join(get_log_folder(), file_name) diff --git a/crazy_functions/批量翻译PDF文档_多线程.py b/crazy_functions/批量翻译PDF文档_多线程.py index 6e9fe6a8..440004e1 100644 --- a/crazy_functions/批量翻译PDF文档_多线程.py +++ b/crazy_functions/批量翻译PDF文档_多线程.py @@ -63,6 +63,7 @@ def 解析PDF_基于GROBID(file_manifest, project_folder, llm_kwargs, plugin_kwa generated_conclusion_files = [] generated_html_files = [] DST_LANG = "中文" + from crazy_functions.crazy_utils import construct_html for index, fp in enumerate(file_manifest): chatbot.append(["当前进度：", f"正在连接GROBID服务，请稍候: {grobid_url}\n如果等待时间过长，请修改config中的GROBID_URL，可修改成本地GROBID服务。"]); yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 article_dict = parse_pdf(fp, grobid_url) @@ -166,6 +167,7 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, TOKEN_LIMIT_PER_FRAGMENT = 1280 generated_conclusion_files = [] generated_html_files = [] + from crazy_functions.crazy_utils import construct_html for index, fp in enumerate(file_manifest): # 读取PDF文件 file_content, page_one = read_and_clean_pdf_text(fp) @@ -261,49 +263,3 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 -class construct_html(): - def __init__(self) -> None: - self.css = """ -.row { - display: flex; - flex-wrap: wrap; -} - -.column { - flex: 1; - padding: 10px; -} - -.table-header { - font-weight: bold; - border-bottom: 1px solid black; -} - -.table-row { - border-bottom: 1px solid lightgray; -} - -.table-cell { - padding: 5px; -} - """ - self.html_string = f'翻译结果' - - - def add_row(self, a, b): - tmp = """ -
-
REPLACE_A
-
REPLACE_B
-
- """ - from toolbox import markdown_convertion - tmp = tmp.replace('REPLACE_A', markdown_convertion(a)) - tmp = tmp.replace('REPLACE_B', markdown_convertion(b)) - self.html_string += tmp - - - def save_file(self, file_name): - with open(os.path.join(get_log_folder(), file_name), 'w', encoding='utf8') as f: - f.write(self.html_string.encode('utf-8', 'ignore').decode()) - return os.path.join(get_log_folder(), file_name) From ec9d030457d46612588bd4f6fe1dca26210d3cf6 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Thu, 14 Sep 2023 00:51:25 +0800 Subject: [PATCH 16/29] =?UTF-8?q?=E6=8A=8A=E4=B8=8A=E4=BC=A0=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E8=B7=AF=E5=BE=84=E5=92=8C=E6=97=A5=E5=BF=97=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E4=BF=AE=E6=94=B9=E4=B8=BA=E7=BB=9F=E4=B8=80=E5=8F=AF?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E7=9A=84=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 5 +- crazy_functional.py | 1 - crazy_functions/CodeInterpreter.py | 9 +- crazy_functions/Latex全文润色.py | 10 +- crazy_functions/Latex全文翻译.py | 7 +- crazy_functions/Latex输出PDF结果.py | 4 +- crazy_functions/latex_fns/latex_actions.py | 8 +- crazy_functions/下载arxiv论文翻译摘要.py | 21 ++-- crazy_functions/代码重写为全英文_多线程.py | 138 --------------------- crazy_functions/图片生成.py | 4 +- crazy_functions/对话历史存档.py | 16 +-- crazy_functions/总结word文档.py | 9 +- crazy_functions/总结音视频.py | 24 ++-- crazy_functions/批量Markdown翻译.py | 4 +- crazy_functions/批量总结PDF文档.py | 7 +- crazy_functions/批量总结PDF文档pdfminer.py | 6 +- crazy_functions/批量翻译PDF文档_多线程.py | 9 +- crazy_functions/生成函数注释.py | 6 +- crazy_functions/虚空终端.py | 4 +- crazy_functions/解析JupyterNotebook.py | 6 +- crazy_functions/解析项目源代码.py | 6 +- crazy_functions/读文章写摘要.py | 39 +++--- crazy_functions/辅助功能.py | 13 +- docs/translate_english.json | 6 - docs/translate_japanese.json | 3 - docs/translate_traditionalchinese.json | 3 - main.py | 10 +- multi_language.py | 6 +- request_llm/bridge_chatgpt.py | 4 +- toolbox.py | 134 ++++++++++---------- 30 files changed, 193 insertions(+), 329 deletions(-) delete mode 100644 crazy_functions/代码重写为全英文_多线程.py diff --git a/config.py b/config.py index 2d57f09b..94a59746 100644 --- a/config.py +++ b/config.py @@ -179,7 +179,10 @@ GROBID_URLS = [ # 是否允许通过自然语言描述修改本页的配置，该功能具有一定的危险性，默认关闭 ALLOW_RESET_CONFIG = False - +# 临时的上传文件夹位置，请勿修改 +PATH_PRIVATE_UPLOAD = "private_upload" +# 日志文件夹的位置，请勿修改 +PATH_LOGGING = "gpt_log" """ 在线大模型配置关联关系示意图 diff --git a/crazy_functional.py b/crazy_functional.py index c5ccda7c..4df53f58 100644 --- a/crazy_functional.py +++ b/crazy_functional.py @@ -13,7 +13,6 @@ def get_crazy_functions(): from crazy_functions.解析项目源代码 import 解析一个Java项目 from crazy_functions.解析项目源代码 import 解析一个前端项目 from crazy_functions.高级功能函数模板 import 高阶功能模板函数 - from crazy_functions.代码重写为全英文_多线程 import 全项目切换英文 from crazy_functions.Latex全文润色 import Latex英文润色 from crazy_functions.询问多个大语言模型 import 同时问询 from crazy_functions.解析项目源代码 import 解析一个Lua项目 diff --git a/crazy_functions/CodeInterpreter.py b/crazy_functions/CodeInterpreter.py index 3c970f35..283dd87a 100644 --- a/crazy_functions/CodeInterpreter.py +++ b/crazy_functions/CodeInterpreter.py @@ -1,6 +1,7 @@ from collections.abc import Callable, Iterable, Mapping from typing import Any -from toolbox import CatchException, update_ui, gen_time_str, trimmed_format_exc, promote_file_to_downloadzone, clear_file_downloadzone +from toolbox import CatchException, update_ui, gen_time_str, trimmed_format_exc +from toolbox import promote_file_to_downloadzone, get_log_folder from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive from .crazy_utils import input_clipping, try_install_deps from multiprocessing import Process, Pipe @@ -92,7 +93,7 @@ def gpt_interact_multi_step(txt, file_type, llm_kwargs, chatbot, history): def make_module(code): module_file = 'gpt_fn_' + gen_time_str().replace('-','_') - with open(f'gpt_log/{module_file}.py', 'w', encoding='utf8') as f: + with open(f'{get_log_folder()}/{module_file}.py', 'w', encoding='utf8') as f: f.write(code) def get_class_name(class_string): @@ -102,7 +103,7 @@ def make_module(code): return class_name class_name = get_class_name(code) - return f"gpt_log.{module_file}->{class_name}" + return f"{get_log_folder().replace('/', '.')}.{module_file}->{class_name}" def init_module_instance(module): import importlib @@ -171,7 +172,7 @@ def 虚空终端CodeInterpreter(txt, llm_kwargs, plugin_kwargs, chatbot, history file_type = file_path.split('.')[-1] # 粗心检查 - if 'private_upload' in txt: + if is_the_upload_folder(txt): chatbot.append([ "...", f"请在输入框内填写需求，然后再次点击该插件（文件路径 {file_path} 已经被记忆）" diff --git a/crazy_functions/Latex全文润色.py b/crazy_functions/Latex全文润色.py index 9aaff6c0..462f9657 100644 --- a/crazy_functions/Latex全文润色.py +++ b/crazy_functions/Latex全文润色.py @@ -1,5 +1,5 @@ -from toolbox import update_ui, trimmed_format_exc -from toolbox import CatchException, report_execption, write_results_to_file, zip_folder +from toolbox import update_ui, trimmed_format_exc, promote_file_to_downloadzone, get_log_folder +from toolbox import CatchException, report_execption, write_history_to_file, zip_folder class PaperFileGroup(): @@ -51,7 +51,7 @@ class PaperFileGroup(): import os, time folder = os.path.dirname(self.file_paths[0]) t = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) - zip_folder(folder, './gpt_log/', f'{t}-polished.zip') + zip_folder(folder, get_log_folder(), f'{t}-polished.zip') def 多文件润色(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en', mode='polish'): @@ -126,7 +126,9 @@ def 多文件润色(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch # <-------- 整理结果，退出 ----------> create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md" - res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name) + res = write_history_to_file(gpt_response_collection, file_basename=create_report_file_name) + promote_file_to_downloadzone(res, chatbot=chatbot) + history = gpt_response_collection chatbot.append((f"{fp}完成了吗？", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/Latex全文翻译.py b/crazy_functions/Latex全文翻译.py index 554c485a..b5aad71b 100644 --- a/crazy_functions/Latex全文翻译.py +++ b/crazy_functions/Latex全文翻译.py @@ -1,5 +1,5 @@ -from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import update_ui, promote_file_to_downloadzone +from toolbox import CatchException, report_execption, write_history_to_file fast_debug = False class PaperFileGroup(): @@ -95,7 +95,8 @@ def 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch # <-------- 整理结果，退出 ----------> create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md" - res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name) + res = write_history_to_file(gpt_response_collection, create_report_file_name) + promote_file_to_downloadzone(res, chatbot=chatbot) history = gpt_response_collection chatbot.append((f"{fp}完成了吗？", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/Latex输出PDF结果.py b/crazy_functions/Latex输出PDF结果.py index 8686f7e9..56ca1c6c 100644 --- a/crazy_functions/Latex输出PDF结果.py +++ b/crazy_functions/Latex输出PDF结果.py @@ -1,4 +1,4 @@ -from toolbox import update_ui, trimmed_format_exc, get_conf, objdump, objload, promote_file_to_downloadzone +from toolbox import update_ui, trimmed_format_exc, get_conf, get_log_folder, promote_file_to_downloadzone from toolbox import CatchException, report_execption, update_ui_lastest_msg, zip_result, gen_time_str from functools import partial import glob, os, requests, time @@ -65,7 +65,7 @@ def move_project(project_folder, arxiv_id=None): if arxiv_id is not None: new_workfolder = pj(ARXIV_CACHE_DIR, arxiv_id, 'workfolder') else: - new_workfolder = f'gpt_log/{gen_time_str()}' + new_workfolder = f'{get_log_folder()}/{gen_time_str()}' try: shutil.rmtree(new_workfolder) except: diff --git a/crazy_functions/latex_fns/latex_actions.py b/crazy_functions/latex_fns/latex_actions.py index 04efc94a..dcde0e99 100644 --- a/crazy_functions/latex_fns/latex_actions.py +++ b/crazy_functions/latex_fns/latex_actions.py @@ -1,4 +1,4 @@ -from toolbox import update_ui, update_ui_lastest_msg # 刷新Gradio前端界面 +from toolbox import update_ui, update_ui_lastest_msg, get_log_folder from toolbox import zip_folder, objdump, objload, promote_file_to_downloadzone from .latex_toolbox import PRESERVE, TRANSFORM from .latex_toolbox import set_forbidden_text, set_forbidden_text_begin_end, set_forbidden_text_careful_brace @@ -439,9 +439,9 @@ def write_html(sp_file_contents, sp_file_result, chatbot, project_folder): trans = k ch.add_row(a=orig, b=trans) create_report_file_name = f"{gen_time_str()}.trans.html" - ch.save_file(create_report_file_name) - shutil.copyfile(pj('./gpt_log/', create_report_file_name), pj(project_folder, create_report_file_name)) - promote_file_to_downloadzone(file=f'./gpt_log/{create_report_file_name}', chatbot=chatbot) + res = ch.save_file(create_report_file_name) + shutil.copyfile(res, pj(project_folder, create_report_file_name)) + promote_file_to_downloadzone(file=res, chatbot=chatbot) except: from toolbox import trimmed_format_exc print('writing html result failed:', trimmed_format_exc()) diff --git a/crazy_functions/下载arxiv论文翻译摘要.py b/crazy_functions/下载arxiv论文翻译摘要.py index 46275bf4..8b4a5037 100644 --- a/crazy_functions/下载arxiv论文翻译摘要.py +++ b/crazy_functions/下载arxiv论文翻译摘要.py @@ -1,5 +1,6 @@ -from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file, get_conf +from toolbox import update_ui, get_log_folder +from toolbox import write_history_to_file, promote_file_to_downloadzone +from toolbox import CatchException, report_execption, get_conf import re, requests, unicodedata, os from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive def download_arxiv_(url_pdf): @@ -28,7 +29,7 @@ def download_arxiv_(url_pdf): if k in other_info['comment']: title = k + ' ' + title - download_dir = './gpt_log/arxiv/' + download_dir = get_log_folder(plugin_name='arxiv') os.makedirs(download_dir, exist_ok=True) title_str = title.replace('?', '？')\ @@ -40,9 +41,6 @@ def download_arxiv_(url_pdf): requests_pdf_url = url_pdf file_path = download_dir+title_str - # if os.path.exists(file_path): - # print('返回缓存文件') - # return './gpt_log/arxiv/'+title_str print('下载中') proxies, = get_conf('proxies') @@ -61,7 +59,7 @@ def download_arxiv_(url_pdf): .replace('\n', '')\ .replace(' ', ' ')\ .replace(' ', ' ') - return './gpt_log/arxiv/'+title_str, other_info + return file_path, other_info def get_name(_url_): @@ -184,11 +182,10 @@ def 下载arxiv论文并翻译摘要(txt, llm_kwargs, plugin_kwargs, chatbot, hi chatbot[-1] = (i_say_show_user, gpt_say) history.append(i_say_show_user); history.append(gpt_say) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 - # 写入文件 - import shutil - # 重置文件的创建时间 - shutil.copyfile(pdf_path, f'./gpt_log/{os.path.basename(pdf_path)}'); os.remove(pdf_path) - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) + promote_file_to_downloadzone(pdf_path, chatbot=chatbot) + chatbot.append(("完成了吗？", res + "\n\nPDF文件也已经下载")) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 diff --git a/crazy_functions/代码重写为全英文_多线程.py b/crazy_functions/代码重写为全英文_多线程.py deleted file mode 100644 index e57f80f1..00000000 --- a/crazy_functions/代码重写为全英文_多线程.py +++ /dev/null @@ -1,138 +0,0 @@ -import threading -from request_llm.bridge_all import predict_no_ui_long_connection -from toolbox import update_ui -from toolbox import CatchException, write_results_to_file, report_execption -from .crazy_utils import breakdown_txt_to_satisfy_token_limit - -def extract_code_block_carefully(txt): - splitted = txt.split('```') - n_code_block_seg = len(splitted) - 1 - if n_code_block_seg <= 1: return txt - # 剩下的情况都开头除去 ``` 结尾除去一次 ``` - txt_out = '```'.join(splitted[1:-1]) - return txt_out - - - -def break_txt_into_half_at_some_linebreak(txt): - lines = txt.split('\n') - n_lines = len(lines) - pre = lines[:(n_lines//2)] - post = lines[(n_lines//2):] - return "\n".join(pre), "\n".join(post) - - -@CatchException -def 全项目切换英文(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys_prompt, web_port): - # 第1步：清空历史，以免输入溢出 - history = [] - - # 第2步：尝试导入依赖，如果缺少依赖，则给出安装建议 - try: - import tiktoken - except: - report_execption(chatbot, history, - a = f"解析项目: {txt}", - b = f"导入软件依赖失败。使用该模块需要额外依赖，安装方法```pip install --upgrade tiktoken```。") - yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - return - - # 第3步：集合文件 - import time, glob, os, shutil, re - os.makedirs('gpt_log/generated_english_version', exist_ok=True) - os.makedirs('gpt_log/generated_english_version/crazy_functions', exist_ok=True) - file_manifest = [f for f in glob.glob('./*.py') if ('test_project' not in f) and ('gpt_log' not in f)] + \ - [f for f in glob.glob('./crazy_functions/*.py') if ('test_project' not in f) and ('gpt_log' not in f)] - # file_manifest = ['./toolbox.py'] - i_say_show_user_buffer = [] - - # 第4步：随便显示点什么防止卡顿的感觉 - for index, fp in enumerate(file_manifest): - # if 'test_project' in fp: continue - with open(fp, 'r', encoding='utf-8', errors='replace') as f: - file_content = f.read() - i_say_show_user =f'[{index}/{len(file_manifest)}] 接下来请将以下代码中包含的所有中文转化为英文，只输出转化后的英文代码，请用代码块输出代码: {os.path.abspath(fp)}' - i_say_show_user_buffer.append(i_say_show_user) - chatbot.append((i_say_show_user, "[Local Message] 等待多线程操作，中间过程不予显示.")) - yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - - - # 第5步：Token限制下的截断与处理 - MAX_TOKEN = 3000 - from request_llm.bridge_all import model_info - enc = model_info["gpt-3.5-turbo"]['tokenizer'] - def get_token_fn(txt): return len(enc.encode(txt, disallowed_special=())) - - - # 第6步：任务函数 - mutable_return = [None for _ in file_manifest] - observe_window = [[""] for _ in file_manifest] - def thread_worker(fp,index): - if index > 10: - time.sleep(60) - print('Openai 限制免费用户每分钟20次请求，降低请求频率中。') - with open(fp, 'r', encoding='utf-8', errors='replace') as f: - file_content = f.read() - i_say_template = lambda fp, file_content: f'接下来请将以下代码中包含的所有中文转化为英文，只输出代码，文件名是{fp}，文件代码是 ```{file_content}```' - try: - gpt_say = "" - # 分解代码文件 - file_content_breakdown = breakdown_txt_to_satisfy_token_limit(file_content, get_token_fn, MAX_TOKEN) - for file_content_partial in file_content_breakdown: - i_say = i_say_template(fp, file_content_partial) - # # ** gpt request ** - gpt_say_partial = predict_no_ui_long_connection(inputs=i_say, llm_kwargs=llm_kwargs, history=[], sys_prompt=sys_prompt, observe_window=observe_window[index]) - gpt_say_partial = extract_code_block_carefully(gpt_say_partial) - gpt_say += gpt_say_partial - mutable_return[index] = gpt_say - except ConnectionAbortedError as token_exceed_err: - print('至少一个线程任务Token溢出而失败', e) - except Exception as e: - print('至少一个线程任务意外失败', e) - - # 第7步：所有线程同时开始执行任务函数 - handles = [threading.Thread(target=thread_worker, args=(fp,index)) for index, fp in enumerate(file_manifest)] - for h in handles: - h.daemon = True - h.start() - chatbot.append(('开始了吗？', f'多线程操作已经开始')) - yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - - # 第8步：循环轮询各个线程是否执行完毕 - cnt = 0 - while True: - cnt += 1 - time.sleep(0.2) - th_alive = [h.is_alive() for h in handles] - if not any(th_alive): break - # 更好的UI视觉效果 - observe_win = [] - for thread_index, alive in enumerate(th_alive): - observe_win.append("[ ..."+observe_window[thread_index][0][-60:].replace('\n','').replace('```','...').replace(' ','.').replace('
','.....').replace('$','.')+"... ]") - stat = [f'执行中: {obs}\n\n' if alive else '已完成\n\n' for alive, obs in zip(th_alive, observe_win)] - stat_str = ''.join(stat) - chatbot[-1] = (chatbot[-1][0], f'多线程操作已经开始，完成情况: \n\n{stat_str}' + ''.join(['.']*(cnt%10+1))) - yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - - # 第9步：把结果写入文件 - for index, h in enumerate(handles): - h.join() # 这里其实不需要join了，肯定已经都结束了 - fp = file_manifest[index] - gpt_say = mutable_return[index] - i_say_show_user = i_say_show_user_buffer[index] - - where_to_relocate = f'gpt_log/generated_english_version/{fp}' - if gpt_say is not None: - with open(where_to_relocate, 'w+', encoding='utf-8') as f: - f.write(gpt_say) - else: # 失败 - shutil.copyfile(file_manifest[index], where_to_relocate) - chatbot.append((i_say_show_user, f'[Local Message] 已完成{os.path.abspath(fp)}的转化，\n\n存入{os.path.abspath(where_to_relocate)}')) - history.append(i_say_show_user); history.append(gpt_say) - yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - time.sleep(1) - - # 第10步：备份一个文件 - res = write_results_to_file(history) - chatbot.append(("生成一份任务执行报告", res)) - yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/图片生成.py b/crazy_functions/图片生成.py index a1cc2441..51a1baff 100644 --- a/crazy_functions/图片生成.py +++ b/crazy_functions/图片生成.py @@ -1,4 +1,4 @@ -from toolbox import CatchException, update_ui, get_conf, select_api_key +from toolbox import CatchException, update_ui, get_conf, select_api_key, get_log_folder from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive import datetime @@ -33,7 +33,7 @@ def gen_image(llm_kwargs, prompt, resolution="256x256"): raise RuntimeError(response.content.decode()) # 文件保存到本地 r = requests.get(image_url, proxies=proxies) - file_path = 'gpt_log/image_gen/' + file_path = f'{get_log_folder()}/image_gen/' os.makedirs(file_path, exist_ok=True) file_name = 'Image' + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '.png' with open(file_path+file_name, 'wb+') as f: f.write(r.content) diff --git a/crazy_functions/对话历史存档.py b/crazy_functions/对话历史存档.py index a0d0ac8e..99bce3d7 100644 --- a/crazy_functions/对话历史存档.py +++ b/crazy_functions/对话历史存档.py @@ -1,4 +1,4 @@ -from toolbox import CatchException, update_ui, promote_file_to_downloadzone +from toolbox import CatchException, update_ui, promote_file_to_downloadzone, get_log_folder from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive import re @@ -10,8 +10,8 @@ def write_chat_to_file(chatbot, history=None, file_name=None): import time if file_name is None: file_name = 'chatGPT对话历史' + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '.html' - os.makedirs('./gpt_log/', exist_ok=True) - with open(f'./gpt_log/{file_name}', 'w', encoding='utf8') as f: + fp = os.path.join(get_log_folder(), file_name) + with open(fp, 'w', encoding='utf8') as f: from themes.theme import advanced_css f.write(f'对话历史') for i, contents in enumerate(chatbot): @@ -29,8 +29,8 @@ def write_chat_to_file(chatbot, history=None, file_name=None): for h in history: f.write("\n>>>" + h) f.write('') - promote_file_to_downloadzone(f'./gpt_log/{file_name}', rename_file=file_name, chatbot=chatbot) - return '对话历史写入：' + os.path.abspath(f'./gpt_log/{file_name}') + promote_file_to_downloadzone(fp, rename_file=file_name, chatbot=chatbot) + return '对话历史写入：' + os.path.abspath(fp) def gen_file_preview(file_name): try: @@ -106,7 +106,7 @@ def 载入对话历史存档(txt, llm_kwargs, plugin_kwargs, chatbot, history, s if not success: if txt == "": txt = '空空如也的输入栏' import glob - local_history = "
".join(["`"+hide_cwd(f)+f" ({gen_file_preview(f)})"+"`" for f in glob.glob(f'gpt_log/**/chatGPT对话历史*.html', recursive=True)]) + local_history = "
".join(["`"+hide_cwd(f)+f" ({gen_file_preview(f)})"+"`" for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True)]) chatbot.append([f"正在查找对话历史文件（html格式）: {txt}", f"找不到任何html文件: {txt}。但本地存储了以下历史文件，您可以将任意一个文件路径粘贴到输入区，然后重试：
{local_history}"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 return @@ -132,8 +132,8 @@ def 删除所有本地对话历史记录(txt, llm_kwargs, plugin_kwargs, chatbot """ import glob, os - local_history = "
".join(["`"+hide_cwd(f)+"`" for f in glob.glob(f'gpt_log/**/chatGPT对话历史*.html', recursive=True)]) - for f in glob.glob(f'gpt_log/**/chatGPT对话历史*.html', recursive=True): + local_history = "
".join(["`"+hide_cwd(f)+"`" for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True)]) + for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True): os.remove(f) chatbot.append([f"删除所有历史对话文件", f"已删除
{local_history}"]) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/总结word文档.py b/crazy_functions/总结word文档.py index 5af69607..4ea753cb 100644 --- a/crazy_functions/总结word文档.py +++ b/crazy_functions/总结word文档.py @@ -1,5 +1,6 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive fast_debug = False @@ -71,11 +72,13 @@ def 解析docx(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot history.extend([i_say,gpt_say]) this_paper_history.extend([i_say,gpt_say]) - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成了吗？", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("所有文件都总结完成了吗？", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/总结音视频.py b/crazy_functions/总结音视频.py index 62f05d39..7c113f47 100644 --- a/crazy_functions/总结音视频.py +++ b/crazy_functions/总结音视频.py @@ -1,5 +1,6 @@ -from toolbox import CatchException, report_execption, select_api_key, update_ui, write_results_to_file, get_conf +from toolbox import CatchException, report_execption, select_api_key, update_ui, get_conf from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive +from toolbox import write_history_to_file, promote_file_to_downloadzone, get_log_folder def split_audio_file(filename, split_duration=1000): """ @@ -15,7 +16,7 @@ def split_audio_file(filename, split_duration=1000): """ from moviepy.editor import AudioFileClip import os - os.makedirs('gpt_log/mp3/cut/', exist_ok=True) # 创建存储切割音频的文件夹 + os.makedirs(f"{get_log_folder(plugin_name='audio')}/mp3/cut/", exist_ok=True) # 创建存储切割音频的文件夹 # 读取音频文件 audio = AudioFileClip(filename) @@ -31,8 +32,8 @@ def split_audio_file(filename, split_duration=1000): start_time = split_points[i] end_time = split_points[i + 1] split_audio = audio.subclip(start_time, end_time) - split_audio.write_audiofile(f"gpt_log/mp3/cut/{filename[0]}_{i}.mp3") - filelist.append(f"gpt_log/mp3/cut/{filename[0]}_{i}.mp3") + split_audio.write_audiofile(f"{get_log_folder(plugin_name='audio')}/mp3/cut/{filename[0]}_{i}.mp3") + filelist.append(f"{get_log_folder(plugin_name='audio')}/mp3/cut/{filename[0]}_{i}.mp3") audio.close() return filelist @@ -52,7 +53,7 @@ def AnalyAudio(parse_prompt, file_manifest, llm_kwargs, chatbot, history): 'Authorization': f"Bearer {api_key}" } - os.makedirs('gpt_log/mp3/', exist_ok=True) + os.makedirs(f"{get_log_folder(plugin_name='audio')}/mp3/", exist_ok=True) for index, fp in enumerate(file_manifest): audio_history = [] # 提取文件扩展名 @@ -60,8 +61,8 @@ def AnalyAudio(parse_prompt, file_manifest, llm_kwargs, chatbot, history): # 提取视频中的音频 if ext not in [".mp3", ".wav", ".m4a", ".mpga"]: audio_clip = AudioFileClip(fp) - audio_clip.write_audiofile(f'gpt_log/mp3/output{index}.mp3') - fp = f'gpt_log/mp3/output{index}.mp3' + audio_clip.write_audiofile(f"{get_log_folder(plugin_name='audio')}/mp3/output{index}.mp3") + fp = f"{get_log_folder(plugin_name='audio')}/mp3/output{index}.mp3" # 调用whisper模型音频转文字 voice = split_audio_file(fp) for j, i in enumerate(voice): @@ -113,18 +114,19 @@ def AnalyAudio(parse_prompt, file_manifest, llm_kwargs, chatbot, history): history=audio_history, sys_prompt="总结文章。" ) - history.extend([i_say, gpt_say]) audio_history.extend([i_say, gpt_say]) - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append((f"第{index + 1}段音频完成了吗？", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 删除中间文件夹 import shutil - shutil.rmtree('gpt_log/mp3') - res = write_results_to_file(history) + shutil.rmtree(f"{get_log_folder(plugin_name='audio')}/mp3") + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("所有音频都总结完成了吗？", res)) yield from update_ui(chatbot=chatbot, history=history) diff --git a/crazy_functions/批量Markdown翻译.py b/crazy_functions/批量Markdown翻译.py index 8af98640..277d942c 100644 --- a/crazy_functions/批量Markdown翻译.py +++ b/crazy_functions/批量Markdown翻译.py @@ -1,7 +1,7 @@ import glob, time, os, re from toolbox import update_ui, trimmed_format_exc, gen_time_str, disable_auto_promotion -from toolbox import CatchException, report_execption, write_history_to_file -from toolbox import promote_file_to_downloadzone, get_log_folder +from toolbox import CatchException, report_execption, get_log_folder +from toolbox import write_history_to_file, promote_file_to_downloadzone fast_debug = False class PaperFileGroup(): diff --git a/crazy_functions/批量总结PDF文档.py b/crazy_functions/批量总结PDF文档.py index fc65f5c8..b87d4825 100644 --- a/crazy_functions/批量总结PDF文档.py +++ b/crazy_functions/批量总结PDF文档.py @@ -1,5 +1,6 @@ from toolbox import update_ui, promote_file_to_downloadzone, gen_time_str -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive from .crazy_utils import read_and_clean_pdf_text from .crazy_utils import input_clipping @@ -99,8 +100,8 @@ do not have too much repetitive information, numerical values using the original _, final_results = input_clipping("", final_results, max_token_limit=3200) yield from update_ui(chatbot=chatbot, history=final_results) # 注意这里的历史记录被替代了 - res = write_results_to_file(file_write_buffer, file_name=gen_time_str()) - promote_file_to_downloadzone(res.split('\t')[-1], chatbot=chatbot) + res = write_history_to_file(file_write_buffer) + promote_file_to_downloadzone(res, chatbot=chatbot) yield from update_ui(chatbot=chatbot, history=final_results) # 刷新界面 diff --git a/crazy_functions/批量总结PDF文档pdfminer.py b/crazy_functions/批量总结PDF文档pdfminer.py index ffbb0559..213d8bb2 100644 --- a/crazy_functions/批量总结PDF文档pdfminer.py +++ b/crazy_functions/批量总结PDF文档pdfminer.py @@ -1,6 +1,7 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive +from toolbox import write_history_to_file, promote_file_to_downloadzone fast_debug = False @@ -115,7 +116,8 @@ def 解析Paper(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo chatbot[-1] = (i_say, gpt_say) history.append(i_say); history.append(gpt_say) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成了吗？", res)) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 diff --git a/crazy_functions/批量翻译PDF文档_多线程.py b/crazy_functions/批量翻译PDF文档_多线程.py index 440004e1..d620715b 100644 --- a/crazy_functions/批量翻译PDF文档_多线程.py +++ b/crazy_functions/批量翻译PDF文档_多线程.py @@ -1,6 +1,6 @@ -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption, get_log_folder from toolbox import update_ui, promote_file_to_downloadzone, update_ui_lastest_msg, disable_auto_promotion -from toolbox import write_history_to_file, get_log_folder +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency from .crazy_utils import read_and_clean_pdf_text @@ -218,10 +218,11 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, final = ["一、论文概况\n\n---\n\n", paper_meta_info.replace('# ', '### ') + '\n\n---\n\n', "二、论文翻译", ""] final.extend(gpt_response_collection_md) create_report_file_name = f"{os.path.basename(fp)}.trans.md" - res = write_results_to_file(final, file_name=create_report_file_name) + res = write_history_to_file(final, create_report_file_name) + promote_file_to_downloadzone(res, chatbot=chatbot) # 更新UI - generated_conclusion_files.append(f'./gpt_log/{create_report_file_name}') + generated_conclusion_files.append(f'{get_log_folder()}/{create_report_file_name}') chatbot.append((f"{fp}完成了吗？", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/生成函数注释.py b/crazy_functions/生成函数注释.py index a564f21d..bf3da6a4 100644 --- a/crazy_functions/生成函数注释.py +++ b/crazy_functions/生成函数注释.py @@ -1,5 +1,6 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive fast_debug = False @@ -27,7 +28,8 @@ def 生成函数注释(file_manifest, project_folder, llm_kwargs, plugin_kwargs, if not fast_debug: time.sleep(2) if not fast_debug: - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成了吗？", res)) yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 diff --git a/crazy_functions/虚空终端.py b/crazy_functions/虚空终端.py index e01b274d..5f33249e 100644 --- a/crazy_functions/虚空终端.py +++ b/crazy_functions/虚空终端.py @@ -46,7 +46,7 @@ explain_msg = """ from pydantic import BaseModel, Field from typing import List -from toolbox import CatchException, update_ui, gen_time_str +from toolbox import CatchException, update_ui, is_the_upload_folder from toolbox import update_ui_lastest_msg, disable_auto_promotion from request_llm.bridge_all import predict_no_ui_long_connection from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive @@ -112,7 +112,7 @@ def 虚空终端(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt # 用简单的关键词检测用户意图 is_certain, _ = analyze_intention_with_simple_rules(txt) - if txt.startswith('private_upload/') and len(txt) == 34: + if is_the_upload_folder(txt): state.set_state(chatbot=chatbot, key='has_provided_explaination', value=False) appendix_msg = "\n\n**很好，您已经上传了文件**，现在请您描述您的需求。" diff --git a/crazy_functions/解析JupyterNotebook.py b/crazy_functions/解析JupyterNotebook.py index b4bcd561..d4a3b49e 100644 --- a/crazy_functions/解析JupyterNotebook.py +++ b/crazy_functions/解析JupyterNotebook.py @@ -1,5 +1,6 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone fast_debug = True @@ -110,7 +111,8 @@ def ipynb解释(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # <-------- 写入文件，退出 ----------> - res = write_results_to_file(history) + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) chatbot.append(("完成了吗？", res)) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 diff --git a/crazy_functions/解析项目源代码.py b/crazy_functions/解析项目源代码.py index debcb51b..ab63bae1 100644 --- a/crazy_functions/解析项目源代码.py +++ b/crazy_functions/解析项目源代码.py @@ -109,9 +109,9 @@ def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, def 解析项目本身(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): history = [] # 清空历史，以免输入溢出 import glob - file_manifest = [f for f in glob.glob('./*.py') if ('test_project' not in f) and ('gpt_log' not in f)] + \ - [f for f in glob.glob('./crazy_functions/*.py') if ('test_project' not in f) and ('gpt_log' not in f)]+ \ - [f for f in glob.glob('./request_llm/*.py') if ('test_project' not in f) and ('gpt_log' not in f)] + file_manifest = [f for f in glob.glob('./*.py')] + \ + [f for f in glob.glob('./crazy_functions/**/*.py')]+ \ + [f for f in glob.glob('./request_llm/**/*.py')] project_folder = './' if len(file_manifest) == 0: report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何python文件: {txt}") diff --git a/crazy_functions/读文章写摘要.py b/crazy_functions/读文章写摘要.py index 72ffe6b1..acdf632c 100644 --- a/crazy_functions/读文章写摘要.py +++ b/crazy_functions/读文章写摘要.py @@ -1,7 +1,7 @@ from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file +from toolbox import CatchException, report_execption +from toolbox import write_history_to_file, promote_file_to_downloadzone from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive -fast_debug = False def 解析Paper(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt): @@ -17,32 +17,29 @@ def 解析Paper(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo chatbot.append((i_say_show_user, "[Local Message] waiting gpt response.")) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - if not fast_debug: - msg = '正常' - # ** gpt request ** - gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say_show_user, llm_kwargs, chatbot, history=[], sys_prompt=system_prompt) # 带超时倒计时 - - chatbot[-1] = (i_say_show_user, gpt_say) - history.append(i_say_show_user); history.append(gpt_say) - yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 - if not fast_debug: time.sleep(2) + msg = '正常' + gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say_show_user, llm_kwargs, chatbot, history=[], sys_prompt=system_prompt) # 带超时倒计时 + chatbot[-1] = (i_say_show_user, gpt_say) + history.append(i_say_show_user); history.append(gpt_say) + yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 + time.sleep(2) all_file = ', '.join([os.path.relpath(fp, project_folder) for index, fp in enumerate(file_manifest)]) i_say = f'根据以上你自己的分析，对全文进行概括，用学术性语言写一段中文摘要，然后再写一段英文摘要（包括{all_file}）。' chatbot.append((i_say, "[Local Message] waiting gpt response.")) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - if not fast_debug: - msg = '正常' - # ** gpt request ** - gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say, llm_kwargs, chatbot, history=history, sys_prompt=system_prompt) # 带超时倒计时 + msg = '正常' + # ** gpt request ** + gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say, llm_kwargs, chatbot, history=history, sys_prompt=system_prompt) # 带超时倒计时 - chatbot[-1] = (i_say, gpt_say) - history.append(i_say); history.append(gpt_say) - yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 - res = write_results_to_file(history) - chatbot.append(("完成了吗？", res)) - yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 + chatbot[-1] = (i_say, gpt_say) + history.append(i_say); history.append(gpt_say) + yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 + res = write_history_to_file(history) + promote_file_to_downloadzone(res, chatbot=chatbot) + chatbot.append(("完成了吗？", res)) + yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面 diff --git a/crazy_functions/辅助功能.py b/crazy_functions/辅助功能.py index e56deaa4..16854e08 100644 --- a/crazy_functions/辅助功能.py +++ b/crazy_functions/辅助功能.py @@ -2,8 +2,8 @@ # @Time : 2023/4/19 # @Author : Spike # @Descr : -from toolbox import update_ui -from toolbox import CatchException, report_execption, write_results_to_file, get_log_folder +from toolbox import update_ui, get_conf +from toolbox import CatchException from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive @@ -30,14 +30,13 @@ def 猜你想问(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt @CatchException def 清除缓存(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): - chatbot.append(['清除本地缓存数据', '执行中. 删除 gpt_log & private_upload']) + chatbot.append(['清除本地缓存数据', '执行中. 删除数据']) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 import shutil, os - gpt_log_dir = os.path.join(os.path.dirname(__file__), '..', 'gpt_log') - private_upload_dir = os.path.join(os.path.dirname(__file__), '..', 'private_upload') - shutil.rmtree(gpt_log_dir, ignore_errors=True) - shutil.rmtree(private_upload_dir, ignore_errors=True) + PATH_PRIVATE_UPLOAD, PATH_LOGGING = get_conf('PATH_PRIVATE_UPLOAD', 'PATH_LOGGING') + shutil.rmtree(PATH_LOGGING, ignore_errors=True) + shutil.rmtree(PATH_PRIVATE_UPLOAD, ignore_errors=True) chatbot.append(['清除本地缓存数据', '执行完成']) yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 \ No newline at end of file diff --git a/docs/translate_english.json b/docs/translate_english.json index a29c3664..91ffaa86 100644 --- a/docs/translate_english.json +++ b/docs/translate_english.json @@ -299,7 +299,6 @@ "地址🚀": "Address 🚀", "感谢热情的": "Thanks to the enthusiastic", "开发者们❤️": "Developers ❤️", - "所有问询记录将自动保存在本地目录./gpt_log/chat_secrets.log": "All inquiry records will be automatically saved in the local directory ./gpt_log/chat_secrets.log", "请注意自我隐私保护哦！": "Please pay attention to self-privacy protection!", "当前模型": "Current model", "输入区": "Input area", @@ -892,7 +891,6 @@ "保存当前对话": "Save current conversation", "您可以调用“LoadConversationHistoryArchive”还原当下的对话": "You can call 'LoadConversationHistoryArchive' to restore the current conversation", "警告！被保存的对话历史可以被使用该系统的任何人查阅": "Warning! The saved conversation history can be viewed by anyone using this system", - "gpt_log/**/chatGPT对话历史*.html": "gpt_log/**/chatGPT conversation history *.html", "正在查找对话历史文件": "Looking for conversation history file", "html格式": "HTML format", "找不到任何html文件": "No HTML files found", @@ -908,7 +906,6 @@ "pip install pywin32 用于doc格式": "pip install pywin32 for doc format", "仅支持Win平台": "Only supports Win platform", "打开文件": "Open file", - "private_upload里面的文件名在解压zip后容易出现乱码": "The file name in private_upload is prone to garbled characters after unzipping", "rar和7z格式正常": "RAR and 7z formats are normal", "故可以只分析文章内容": "So you can only analyze the content of the article", "不输入文件名": "Do not enter the file name", @@ -1364,7 +1361,6 @@ "注意文章中的每一句话都要翻译": "Please translate every sentence in the article", "一、论文概况": "I. Overview of the paper", "二、论文翻译": "II. Translation of the paper", - "/gpt_log/总结论文-": "/gpt_log/Summary of the paper-", "给出输出文件清单": "Provide a list of output files", "第 0 步": "Step 0", "切割PDF": "Split PDF", @@ -1564,7 +1560,6 @@ "广义速度": "Generalized velocity", "粒子的固有": "Intrinsic of particle", "一个包含所有切割音频片段文件路径的列表": "A list containing the file paths of all segmented audio clips", - "/gpt_log/翻译-": "Translation log-", "计算文件总时长和切割点": "Calculate total duration and cutting points of the file", "总结音频": "Summarize audio", "作者": "Author", @@ -2339,7 +2334,6 @@ "将文件拖动到文件上传区": "Drag and drop the file to the file upload area", "如果意图模糊": "If the intent is ambiguous", "星火认知大模型": "Spark Cognitive Big Model", - "执行中. 删除 gpt_log & private_upload": "Executing. Delete gpt_log & private_upload", "默认 Color = secondary": "Default Color = secondary", "此处也不需要修改": "No modification is needed here", "⭐ ⭐ ⭐ 分析用户意图": "⭐ ⭐ ⭐ Analyze user intent", diff --git a/docs/translate_japanese.json b/docs/translate_japanese.json index 15a4b319..ecd370e9 100644 --- a/docs/translate_japanese.json +++ b/docs/translate_japanese.json @@ -301,7 +301,6 @@ "缺少的依赖": "不足している依存関係", "紫色": "紫色", "唤起高级参数输入区": "高度なパラメータ入力エリアを呼び出す", - "所有问询记录将自动保存在本地目录./gpt_log/chat_secrets.log": "すべての問い合わせ記録は自動的にローカルディレクトリ./gpt_log/chat_secrets.logに保存されます", "则换行符更有可能表示段落分隔": "したがって、改行記号は段落の区切りを表す可能性がより高いです", "；4、引用数量": "；4、引用数量", "中转网址预览": "中継ウェブサイトのプレビュー", @@ -448,7 +447,6 @@ "表示函数是否成功执行": "関数が正常に実行されたかどうかを示す", "一般原样传递下去就行": "通常はそのまま渡すだけでよい", "琥珀色": "琥珀色", - "gpt_log/**/chatGPT对话历史*.html": "gpt_log/**/chatGPT対話履歴*.html", "jittorllms 没有 sys_prompt 接口": "jittorllmsにはsys_promptインターフェースがありません", "清除": "クリア", "小于正文的": "本文より小さい", @@ -1234,7 +1232,6 @@ "找不到任何前端相关文件": "No frontend-related files can be found", "Not enough point. API2D账户点数不足": "Not enough points. API2D account points are insufficient", "当前版本": "Current version", - "/gpt_log/总结论文-": "/gpt_log/Summary paper-", "1. 临时解决方案": "1. Temporary solution", "第8步": "Step 8", "历史": "History", diff --git a/docs/translate_traditionalchinese.json b/docs/translate_traditionalchinese.json index cf12de1c..efaa62fc 100644 --- a/docs/translate_traditionalchinese.json +++ b/docs/translate_traditionalchinese.json @@ -314,7 +314,6 @@ "请用markdown格式输出": "請用 Markdown 格式輸出", "模仿ChatPDF": "模仿 ChatPDF", "等待多久判定为超时": "等待多久判定為超時", - "/gpt_log/总结论文-": "/gpt_log/總結論文-", "请结合互联网信息回答以下问题": "請結合互聯網信息回答以下問題", "IP查询频率受限": "IP查詢頻率受限", "高级参数输入区的显示提示": "高級參數輸入區的顯示提示", @@ -511,7 +510,6 @@ "將生成的報告自動投射到文件上傳區": "將生成的報告自動上傳到文件區", "函數插件作者": "函數插件作者", "將要匹配的模式": "將要匹配的模式", - "所有问询记录将自动保存在本地目录./gpt_log/chat_secrets.log": "所有詢問記錄將自動保存在本地目錄./gpt_log/chat_secrets.log", "正在分析一个项目的源代码": "正在分析一個專案的源代碼", "使每个段落之间有两个换行符分隔": "使每個段落之間有兩個換行符分隔", "并在被装饰的函数上执行": "並在被裝飾的函數上執行", @@ -1059,7 +1057,6 @@ "重试中": "重試中", "月": "月份", "localhost意思是代理软件安装在本机上": "localhost意思是代理軟體安裝在本機上", - "gpt_log/**/chatGPT对话历史*.html": "gpt_log/**/chatGPT對話歷史*.html", "的长度必须小于 2500 个 Token": "長度必須小於 2500 個 Token", "抽取可用的api-key": "提取可用的api-key", "增强报告的可读性": "增強報告的可讀性", diff --git a/main.py b/main.py index 628d30b9..aedbcfc0 100644 --- a/main.py +++ b/main.py @@ -8,7 +8,7 @@ def main(): # 建议您复制一个config_private.py放自己的秘密, 如API和代理网址, 避免不小心传github被别人看到 proxies, WEB_PORT, LLM_MODEL, CONCURRENT_COUNT, AUTHENTICATION = get_conf('proxies', 'WEB_PORT', 'LLM_MODEL', 'CONCURRENT_COUNT', 'AUTHENTICATION') CHATBOT_HEIGHT, LAYOUT, AVAIL_LLM_MODELS, AUTO_CLEAR_TXT = get_conf('CHATBOT_HEIGHT', 'LAYOUT', 'AVAIL_LLM_MODELS', 'AUTO_CLEAR_TXT') - ENABLE_AUDIO, AUTO_CLEAR_TXT = get_conf('ENABLE_AUDIO', 'AUTO_CLEAR_TXT') + ENABLE_AUDIO, AUTO_CLEAR_TXT, PATH_LOGGING = get_conf('ENABLE_AUDIO', 'AUTO_CLEAR_TXT', 'PATH_LOGGING') # 如果WEB_PORT是-1, 则随机选取WEB端口 PORT = find_free_port() if WEB_PORT <= 0 else WEB_PORT @@ -21,12 +21,12 @@ def main(): # 问询记录, python 版本建议3.9+（越新越好） import logging, uuid - os.makedirs("gpt_log", exist_ok=True) - try:logging.basicConfig(filename="gpt_log/chat_secrets.log", level=logging.INFO, encoding="utf-8", format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") - except:logging.basicConfig(filename="gpt_log/chat_secrets.log", level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") + os.makedirs(PATH_LOGGING, exist_ok=True) + try:logging.basicConfig(filename=f"{PATH_LOGGING}/chat_secrets.log", level=logging.INFO, encoding="utf-8", format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") + except:logging.basicConfig(filename=f"{PATH_LOGGING}/chat_secrets.log", level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S") # Disable logging output from the 'httpx' logger logging.getLogger("httpx").setLevel(logging.WARNING) - print("所有问询记录将自动保存在本地目录./gpt_log/chat_secrets.log, 请注意自我隐私保护哦！") + print(f"所有问询记录将自动保存在本地目录./{PATH_LOGGING}/chat_secrets.log, 请注意自我隐私保护哦！") # 一些普通功能模块 from core_functional import get_core_functions diff --git a/multi_language.py b/multi_language.py index c4ed36eb..8e3ac9d7 100644 --- a/multi_language.py +++ b/multi_language.py @@ -33,9 +33,11 @@ import functools import re import pickle import time +from toolbox import get_conf -CACHE_FOLDER = "gpt_log" -blacklist = ['multi-language', 'gpt_log', '.git', 'private_upload', 'multi_language.py', 'build', '.github', '.vscode', '__pycache__', 'venv'] +CACHE_FOLDER, = get_conf('PATH_LOGGING') + +blacklist = ['multi-language', CACHE_FOLDER, '.git', 'private_upload', 'multi_language.py', 'build', '.github', '.vscode', '__pycache__', 'venv'] # LANG = "TraditionalChinese" # TransPrompt = f"Replace each json value `#` with translated results in Traditional Chinese, e.g., \"原始文本\":\"翻譯後文字\". Keep Json format. Do not answer #." diff --git a/request_llm/bridge_chatgpt.py b/request_llm/bridge_chatgpt.py index 19d10022..6a458f20 100644 --- a/request_llm/bridge_chatgpt.py +++ b/request_llm/bridge_chatgpt.py @@ -21,7 +21,7 @@ import importlib # config_private.py放自己的秘密如API和代理网址 # 读取时首先看是否存在私密的config_private配置文件（不受git管控），如果有，则覆盖原config文件 -from toolbox import get_conf, update_ui, is_any_api_key, select_api_key, what_keys, clip_history, trimmed_format_exc +from toolbox import get_conf, update_ui, is_any_api_key, select_api_key, what_keys, clip_history, trimmed_format_exc, is_the_upload_folder proxies, TIMEOUT_SECONDS, MAX_RETRY, API_ORG = \ get_conf('proxies', 'TIMEOUT_SECONDS', 'MAX_RETRY', 'API_ORG') @@ -138,7 +138,7 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp yield from update_ui(chatbot=chatbot, history=history, msg="等待响应") # 刷新界面 # check mis-behavior - if raw_input.startswith('private_upload/') and len(raw_input) == 34: + if is_the_upload_folder(raw_input): chatbot[-1] = (inputs, f"[Local Message] 检测到操作错误！当您上传文档之后，需要点击“函数插件区”按钮进行处理，而不是点击“提交”按钮。") yield from update_ui(chatbot=chatbot, history=history, msg="正常") # 刷新界面 time.sleep(2) diff --git a/toolbox.py b/toolbox.py index ab8d046a..947044fc 100644 --- a/toolbox.py +++ b/toolbox.py @@ -5,6 +5,8 @@ import inspect import re import os import gradio +import shutil +import glob from latex2mathml.converter import convert as tex2mathml from functools import wraps, lru_cache pj = os.path.join @@ -171,7 +173,7 @@ def HotReload(f): ======================================================================== 第二部分 其他小工具: - - write_results_to_file: 将结果写入markdown文件中 + - write_history_to_file: 将结果写入markdown文件中 - regular_txt_to_markdown: 将普通文本转换为Markdown格式的文本。 - report_execption: 向chatbot中添加简单的意外错误信息 - text_divide_paragraph: 将文本按照段落分隔符分割开，生成带有段落标签的HTML代码。 @@ -203,36 +205,6 @@ def get_reduce_token_percent(text): return 0.5, '不详' -def write_results_to_file(history, file_name=None): - """ - 将对话记录history以Markdown格式写入文件中。如果没有指定文件名，则使用当前时间生成文件名。 - """ - import os - import time - if file_name is None: - # file_name = time.strftime("chatGPT分析报告%Y-%m-%d-%H-%M-%S", time.localtime()) + '.md' - file_name = 'GPT-Report-' + gen_time_str() + '.md' - os.makedirs('./gpt_log/', exist_ok=True) - with open(f'./gpt_log/{file_name}', 'w', encoding='utf8') as f: - f.write('# GPT-Academic Report\n') - for i, content in enumerate(history): - try: - if type(content) != str: content = str(content) - except: - continue - if i % 2 == 0: - f.write('## ') - try: - f.write(content) - except: - # remove everything that cannot be handled by utf8 - f.write(content.encode('utf-8', 'ignore').decode()) - f.write('\n\n') - res = '以上材料已经被写入:\t' + os.path.abspath(f'./gpt_log/{file_name}') - print(res) - return res - - def write_history_to_file(history, file_basename=None, file_fullname=None): """ 将对话记录history以Markdown格式写入文件中。如果没有指定文件名，则使用当前时间生成文件名。 @@ -241,9 +213,9 @@ def write_history_to_file(history, file_basename=None, file_fullname=None): import time if file_fullname is None: if file_basename is not None: - file_fullname = os.path.join(get_log_folder(), file_basename) + file_fullname = pj(get_log_folder(), file_basename) else: - file_fullname = os.path.join(get_log_folder(), f'GPT-Academic-{gen_time_str()}.md') + file_fullname = pj(get_log_folder(), f'GPT-Academic-{gen_time_str()}.md') os.makedirs(os.path.dirname(file_fullname), exist_ok=True) with open(file_fullname, 'w', encoding='utf8') as f: f.write('# GPT-Academic Report\n') @@ -519,7 +491,7 @@ def find_recent_files(directory): if not os.path.exists(directory): os.makedirs(directory, exist_ok=True) for filename in os.listdir(directory): - file_path = os.path.join(directory, filename) + file_path = pj(directory, filename) if file_path.endswith('.log'): continue created_time = os.path.getmtime(file_path) @@ -534,7 +506,7 @@ def promote_file_to_downloadzone(file, rename_file=None, chatbot=None): # 将文件复制一份到下载区 import shutil if rename_file is None: rename_file = f'{gen_time_str()}-{os.path.basename(file)}' - new_path = os.path.join(get_log_folder(), rename_file) + new_path = pj(get_log_folder(), rename_file) # 如果已经存在，先删除 if os.path.exists(new_path) and not os.path.samefile(new_path, file): os.remove(new_path) # 把文件复制过去 @@ -549,44 +521,70 @@ def disable_auto_promotion(chatbot): chatbot._cookies.update({'files_to_promote': []}) return -def on_file_uploaded(files, chatbot, txt, txt2, checkboxes, cookies): +def is_the_upload_folder(string): + PATH_PRIVATE_UPLOAD, = get_conf('PATH_PRIVATE_UPLOAD') + pattern = r'^PATH_PRIVATE_UPLOAD/[A-Za-z0-9_-]+/\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}$' + pattern = pattern.replace('PATH_PRIVATE_UPLOAD', PATH_PRIVATE_UPLOAD) + if re.match(pattern, string): return True + else: return False + +def del_outdated_uploads(outdate_time_seconds): + PATH_PRIVATE_UPLOAD, = get_conf('PATH_PRIVATE_UPLOAD') + current_time = time.time() + one_hour_ago = current_time - outdate_time_seconds + # Get a list of all subdirectories in the PATH_PRIVATE_UPLOAD folder + # Remove subdirectories that are older than one hour + for subdirectory in glob.glob(f'{PATH_PRIVATE_UPLOAD}/*/*'): + subdirectory_time = os.path.getmtime(subdirectory) + if subdirectory_time < one_hour_ago: + try: shutil.rmtree(subdirectory) + except: pass + return + +def on_file_uploaded(request: gradio.Request, files, chatbot, txt, txt2, checkboxes, cookies): """ 当文件被上传时的回调函数 """ if len(files) == 0: return chatbot, txt - import shutil - import os - import time - import glob - from toolbox import extract_archive - try: - shutil.rmtree('./private_upload/') - except: - pass + + # 移除过时的旧文件从而节省空间&保护隐私 + outdate_time_seconds = 60 + del_outdated_uploads(outdate_time_seconds) + + # 创建工作路径 + user_name = "default" if not request.username else request.username time_tag = gen_time_str() - os.makedirs(f'private_upload/{time_tag}', exist_ok=True) - err_msg = '' + PATH_PRIVATE_UPLOAD, = get_conf('PATH_PRIVATE_UPLOAD') + target_path_base = pj(PATH_PRIVATE_UPLOAD, user_name, time_tag) + os.makedirs(target_path_base, exist_ok=True) + + # 逐个文件转移到目标路径 + upload_msg = '' for file in files: file_origin_name = os.path.basename(file.orig_name) - shutil.copy(file.name, f'private_upload/{time_tag}/{file_origin_name}') - err_msg += extract_archive(f'private_upload/{time_tag}/{file_origin_name}', - dest_dir=f'private_upload/{time_tag}/{file_origin_name}.extract') - moved_files = [fp for fp in glob.glob('private_upload/**/*', recursive=True)] - if "底部输入区" in checkboxes: - txt = "" - txt2 = f'private_upload/{time_tag}' + this_file_path = pj(target_path_base, file_origin_name) + shutil.move(file.name, this_file_path) + upload_msg += extract_archive(file_path=this_file_path, dest_dir=this_file_path+'.extract') + + # 整理文件集合 + moved_files = [fp for fp in glob.glob(f'{target_path_base}/**/*', recursive=True)] + if "底部输入区" in checkboxes: + txt, txt2 = "", target_path_base else: - txt = f'private_upload/{time_tag}' - txt2 = "" + txt, txt2 = target_path_base, "" + + # 输出消息 moved_files_str = '\t\n\n'.join(moved_files) - chatbot.append(['我上传了文件，请查收', + chatbot.append(['我上传了文件，请查收', f'[Local Message] 收到以下文件: \n\n{moved_files_str}' + f'\n\n调用路径参数已自动修正到: \n\n{txt}' + - f'\n\n现在您点击任意函数插件时，以上文件将被作为输入参数'+err_msg]) + f'\n\n现在您点击任意函数插件时，以上文件将被作为输入参数'+upload_msg]) + + # 记录近期文件 cookies.update({ 'most_recent_uploaded': { - 'path': f'private_upload/{time_tag}', + 'path': target_path_base, 'time': time.time(), 'time_str': time_tag }}) @@ -595,11 +593,12 @@ def on_file_uploaded(files, chatbot, txt, txt2, checkboxes, cookies): def on_report_generated(cookies, files, chatbot): from toolbox import find_recent_files + PATH_LOGGING, = get_conf('PATH_LOGGING') if 'files_to_promote' in cookies: report_files = cookies['files_to_promote'] cookies.pop('files_to_promote') else: - report_files = find_recent_files('gpt_log') + report_files = find_recent_files(PATH_LOGGING) if len(report_files) == 0: return cookies, None, chatbot # files.extend(report_files) @@ -909,34 +908,35 @@ def zip_folder(source_folder, dest_folder, zip_name): return # Create the name for the zip file - zip_file = os.path.join(dest_folder, zip_name) + zip_file = pj(dest_folder, zip_name) # Create a ZipFile object with zipfile.ZipFile(zip_file, 'w', zipfile.ZIP_DEFLATED) as zipf: # Walk through the source folder and add files to the zip file for foldername, subfolders, filenames in os.walk(source_folder): for filename in filenames: - filepath = os.path.join(foldername, filename) + filepath = pj(foldername, filename) zipf.write(filepath, arcname=os.path.relpath(filepath, source_folder)) # Move the zip file to the destination folder (if it wasn't already there) if os.path.dirname(zip_file) != dest_folder: - os.rename(zip_file, os.path.join(dest_folder, os.path.basename(zip_file))) - zip_file = os.path.join(dest_folder, os.path.basename(zip_file)) + os.rename(zip_file, pj(dest_folder, os.path.basename(zip_file))) + zip_file = pj(dest_folder, os.path.basename(zip_file)) print(f"Zip file created at {zip_file}") def zip_result(folder): t = gen_time_str() - zip_folder(folder, './gpt_log/', f'{t}-result.zip') - return pj('./gpt_log/', f'{t}-result.zip') + zip_folder(folder, get_log_folder(), f'{t}-result.zip') + return pj(get_log_folder(), f'{t}-result.zip') def gen_time_str(): import time return time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) def get_log_folder(user='default', plugin_name='shared'): - _dir = os.path.join(os.path.dirname(__file__), 'gpt_log', user, plugin_name) + PATH_LOGGING, = get_conf('PATH_LOGGING') + _dir = pj(PATH_LOGGING, user, plugin_name) if not os.path.exists(_dir): os.makedirs(_dir) return _dir From 544771db9a7d7ac2ab9061f764cb1d3e1308b375 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Thu, 14 Sep 2023 00:53:15 +0800 Subject: [PATCH 17/29] =?UTF-8?q?=E9=9A=90=E8=97=8F=E5=8E=86=E5=8F=B2?= =?UTF-8?q?=E5=AF=B9=E8=AF=9D=E7=BB=9D=E5=AF=B9=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/对话历史存档.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crazy_functions/对话历史存档.py b/crazy_functions/对话历史存档.py index 99bce3d7..f89faeda 100644 --- a/crazy_functions/对话历史存档.py +++ b/crazy_functions/对话历史存档.py @@ -30,7 +30,7 @@ def write_chat_to_file(chatbot, history=None, file_name=None): f.write("\n>>>" + h) f.write('') promote_file_to_downloadzone(fp, rename_file=file_name, chatbot=chatbot) - return '对话历史写入：' + os.path.abspath(fp) + return '对话历史写入：' + fp def gen_file_preview(file_name): try: From ab7999c71ab1fdbff9f72773af565466608c7286 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Thu, 14 Sep 2023 01:00:38 +0800 Subject: [PATCH 18/29] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E6=9C=AC=E9=A1=B9?= =?UTF-8?q?=E7=9B=AE=E6=BA=90=E7=A0=81=E8=8C=83=E5=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/解析项目源代码.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crazy_functions/解析项目源代码.py b/crazy_functions/解析项目源代码.py index ab63bae1..10333629 100644 --- a/crazy_functions/解析项目源代码.py +++ b/crazy_functions/解析项目源代码.py @@ -23,7 +23,7 @@ def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs, file_content = f.read() prefix = "接下来请你逐文件分析下面的工程" if index==0 else "" i_say = prefix + f'请对下面的程序文件做一个概述文件名是{os.path.relpath(fp, project_folder)}，文件代码是 ```{file_content}```' - i_say_show_user = prefix + f'[{index}/{len(file_manifest)}] 请对下面的程序文件做一个概述: {os.path.abspath(fp)}' + i_say_show_user = prefix + f'[{index}/{len(file_manifest)}] 请对下面的程序文件做一个概述: {fp}' # 装载请求内容 inputs_array.append(i_say) inputs_show_user_array.append(i_say_show_user) @@ -110,8 +110,7 @@ def 解析项目本身(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_ history = [] # 清空历史，以免输入溢出 import glob file_manifest = [f for f in glob.glob('./*.py')] + \ - [f for f in glob.glob('./crazy_functions/**/*.py')]+ \ - [f for f in glob.glob('./request_llm/**/*.py')] + [f for f in glob.glob('./*/*.py')] project_folder = './' if len(file_manifest) == 0: report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何python文件: {txt}") From 74f35e340103d4db0914fadf237a16540470d3fb Mon Sep 17 00:00:00 2001 From: binary-husky Date: Thu, 14 Sep 2023 01:51:55 +0800 Subject: [PATCH 19/29] =?UTF-8?q?=E9=92=88=E5=AF=B9=E8=99=9A=E7=A9=BA?= =?UTF-8?q?=E7=BB=88=E7=AB=AF=E4=B8=AA=E5=88=AB=E6=83=85=E5=86=B5=E4=B8=8B?= =?UTF-8?q?=E4=B8=8D=E8=BE=93=E5=87=BA=E6=96=87=E4=BB=B6=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98=E8=BF=9B=E8=A1=8C=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/translate_english.json | 115 ++++++++++++++++++++++-------------- main.py | 4 +- toolbox.py | 16 ++++- 3 files changed, 86 insertions(+), 49 deletions(-) diff --git a/docs/translate_english.json b/docs/translate_english.json index 91ffaa86..91eba9d3 100644 --- a/docs/translate_english.json +++ b/docs/translate_english.json @@ -2443,48 +2443,75 @@ "├── CODE_HIGHLIGHT 代码高亮": "├── CODE_HIGHLIGHT Code highlighting", "记得用插件": "Remember to use the plugin", "谨慎操作": "Handle with caution", - "请检查PDF是否损坏": "#", - "执行成功了": "#", - "请在输入框内填写需求": "#", - "结果": "#", - "开始干正事": "#", - "次代码生成尝试": "#", - "代码生成结束": "#", - "Nougat解析论文失败": "#", - "受到google限制": "#", - "收尾": "#", - "结果是一个有效文件": "#", - "然后再次点击该插件": "#", - "用插件实现」": "#", - "文件路径": "#", - "仅供测试": "#", - "将csv文件转excel表格": "#", - "开始执行": "#", - "测试": "#", - "睡一会防止触发google反爬虫": "#", - "某段话的整个句子": "#", - "使用tex格式公式 测试2 给出柯西不等式": "#", - "找不到本地项目或无法处理": "#", - "交换图像的蓝色通道和红色通道": "#", - "第三步": "#", - "返回给定的url解析出的arxiv_id": "#", - "裁剪图像": "#", - "已经被记忆": "#", - "无法从bing获取信息！": "#", - "可能触发了google反爬虫机制": "#", - "检索文章的历史版本的题目": "#", - "请配置讯飞星火大模型的XFYUN_APPID": "#", - "执行失败了": "#", - "需要花费较长时间下载NOUGAT参数": "#", - "请检查": "#", - "写入": "#", - "下个句子中已经说完的部分": "#", - "精准翻译PDF文档": "#", - "解析python源代码项目": "#", - "首先在arxiv上搜索": "#", - "错误追踪": "#", - "结果是一个字符串": "#", - "由 test_on_sentence_end": "#", - "获取文章摘要": "#", - "受到bing限制": "#" + "private_upload里面的文件名在解压zip后容易出现乱码": "The file name inside private_upload is prone to garbled characters after unzipping", + "直接返回报错": "Direct return error", + "临时的上传文件夹位置": "Temporary upload folder location", + "使用latex格式 测试3 写出麦克斯韦方程组": "Write Maxwell's equations using latex format for test 3", + "这是一张图片": "This is an image", + "没有发现任何近期上传的文件": "No recent uploaded files found", + "如url未成功匹配返回None": "Return None if the URL does not match successfully", + "如果有Latex环境": "If there is a Latex environment", + "第一次运行时": "When running for the first time", + "创建工作路径": "Create a working directory", + "向": "To", + "执行中. 删除数据": "Executing. Deleting data", + "CodeInterpreter开源版": "CodeInterpreter open source version", + "建议选择更稳定的接口": "It is recommended to choose a more stable interface", + "现在您点击任意函数插件时": "Now when you click on any function plugin", + "请使用“LatexEnglishCorrection+高亮”插件": "Please use the 'LatexEnglishCorrection+Highlight' plugin", + "安装完成": "Installation completed", + "记得用插件！」": "Remember to use the plugin!", + "结论": "Conclusion", + "无法下载资源": "Unable to download resources", + "首先排除一个one-api没有done数据包的第三方Bug情形": "First exclude a third-party bug where one-api does not have a done data package", + "知识库中添加文件": "Add files to the knowledge base", + "处理重名的章节": "Handling duplicate chapter names", + "先上传文件素材": "Upload file materials first", + "无法从google获取信息！": "Unable to retrieve information from Google!", + "展示如下": "Display as follows", + "「把Arxiv论文翻译成中文PDF": "Translate Arxiv papers into Chinese PDF", + "论文我刚刚放到上传区了」": "I just put the paper in the upload area", + "正在下载Gradio主题": "Downloading Gradio themes", + "再运行此插件": "Run this plugin again", + "记录近期文件": "Record recent files", + "粗心检查": "Careful check", + "更多主题": "More themes", + "//huggingface.co/spaces/gradio/theme-gallery 可选": "//huggingface.co/spaces/gradio/theme-gallery optional", + "由 test_on_result_chg": "By test_on_result_chg", + "所有问询记录将自动保存在本地目录./": "All inquiry records will be automatically saved in the local directory ./", + "正在解析论文": "Analyzing the paper", + "逐个文件转移到目标路径": "Move each file to the target path", + "最多重试5次": "Retry up to 5 times", + "日志文件夹的位置": "Location of the log folder", + "我们暂时无法解析此PDF文档": "We are temporarily unable to parse this PDF document", + "文件检索": "File retrieval", + "/**/chatGPT对话历史*.html": "/**/chatGPT conversation history*.html", + "非OpenAI官方接口返回了错误": "Non-OpenAI official interface returned an error", + "如果在Arxiv上匹配失败": "If the match fails on Arxiv", + "文件进入知识库后可长期保存": "Files can be saved for a long time after entering the knowledge base", + "您可以再次重试": "You can try again", + "整理文件集合": "Organize file collection", + "检测到有缺陷的非OpenAI官方接口": "Detected defective non-OpenAI official interface", + "此插件不调用Latex": "This plugin does not call Latex", + "移除过时的旧文件从而节省空间&保护隐私": "Remove outdated old files to save space & protect privacy", + "代码我刚刚打包拖到上传区了」": "I just packed the code and dragged it to the upload area", + "将图像转为灰度图像": "Convert the image to grayscale", + "待排除": "To be excluded", + "请勿修改": "Please do not modify", + "crazy_functions/代码重写为全英文_多线程.py": "crazy_functions/code rewritten to all English_multi-threading.py", + "开发中": "Under development", + "请查阅Gradio主题商店": "Please refer to the Gradio theme store", + "输出消息": "Output message", + "其他情况": "Other situations", + "获取文献失败": "Failed to retrieve literature", + "可以通过再次调用本插件的方式": "You can use this plugin again by calling it", + "保留下半部分": "Keep the lower half", + "排除问题": "Exclude the problem", + "知识库": "Knowledge base", + "ParsePDF失败": "ParsePDF failed", + "向知识库追加更多文档": "Append more documents to the knowledge base", + "此处待注入的知识库名称id": "The knowledge base name ID to be injected here", + "您需要构建知识库后再运行此插件": "You need to build the knowledge base before running this plugin", + "判定是否为公式 | 测试1 写出洛伦兹定律": "Determine whether it is a formula | Test 1 write out the Lorentz law", + "构建知识库后": "After building the knowledge base" } \ No newline at end of file diff --git a/main.py b/main.py index aedbcfc0..ff4bc78d 100644 --- a/main.py +++ b/main.py @@ -184,7 +184,7 @@ def main(): # 函数插件-固定按钮区 for k in plugins: if not plugins[k].get("AsButton", True): continue - click_handle = plugins[k]["Button"].click(ArgsGeneralWrapper(plugins[k]["Function"]), [*input_combo, gr.State(PORT)], output_combo) + click_handle = plugins[k]["Button"].click(ArgsGeneralWrapper(plugins[k]["Function"]), [*input_combo], output_combo) click_handle.then(on_report_generated, [cookies, file_upload, chatbot], [cookies, file_upload, chatbot]) cancel_handles.append(click_handle) # 函数插件-下拉菜单与随变按钮的互动 @@ -204,7 +204,7 @@ def main(): def route(request: gr.Request, k, *args, **kwargs): if k in [r"打开插件列表", r"请先从插件列表中选择"]: return yield from ArgsGeneralWrapper(plugins[k]["Function"])(request, *args, **kwargs) - click_handle = switchy_bt.click(route,[switchy_bt, *input_combo, gr.State(PORT)], output_combo) + click_handle = switchy_bt.click(route,[switchy_bt, *input_combo], output_combo) click_handle.then(on_report_generated, [cookies, file_upload, chatbot], [cookies, file_upload, chatbot]) cancel_handles.append(click_handle) # 终止按钮的回调函数注册 diff --git a/toolbox.py b/toolbox.py index 947044fc..ccc36643 100644 --- a/toolbox.py +++ b/toolbox.py @@ -79,14 +79,23 @@ def ArgsGeneralWrapper(f): } chatbot_with_cookie = ChatBotWithCookies(cookies) chatbot_with_cookie.write_list(chatbot) + if cookies.get('lock_plugin', None) is None: # 正常状态 - yield from f(txt_passon, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, system_prompt, *args) + if len(args) == 0: # 插件通道 + yield from f(txt_passon, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, system_prompt, request) + else: # 对话通道 + yield from f(txt_passon, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, system_prompt, *args) else: - # 处理个别特殊插件的锁定状态 + # 处理少数情况下的特殊插件的锁定状态 module, fn_name = cookies['lock_plugin'].split('->') f_hot_reload = getattr(importlib.import_module(module, fn_name), fn_name) yield from f_hot_reload(txt_passon, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, system_prompt, request) + # 判断一下用户是否错误地通过对话通道进入，如果是，则进行提醒 + final_cookies = chatbot_with_cookie.get_cookies() + if len(args) == 0 and 'files_to_promote' in final_cookies and len(final_cookies['files_to_promote']) > 0: + chatbot_with_cookie.append(["检测到**滞留的缓存文档**，请及时处理。", "请及时点击“**保存当前对话**”获取所有滞留文档。"]) + yield from update_ui(chatbot_with_cookie, final_cookies['history'], msg="检测到被滞留的缓存文档") return decorated @@ -96,7 +105,8 @@ def update_ui(chatbot, history, msg='正常', **kwargs): # 刷新界面 """ assert isinstance(chatbot, ChatBotWithCookies), "在传递chatbot的过程中不要将其丢弃。必要时, 可用clear将其清空, 然后用for+append循环重新赋值。" cookies = chatbot.get_cookies() - + # 备份一份History作为记录 + cookies.update({'history': history}) # 解决插件锁定时的界面显示问题 if cookies.get('lock_plugin', None): label = cookies.get('llm_model', "") + " | " + "正在锁定插件" + cookies.get('lock_plugin', None) From 0937f373885800513e37b75442067e2f6026b0a8 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Thu, 14 Sep 2023 11:02:40 +0800 Subject: [PATCH 20/29] =?UTF-8?q?Predict=E6=8C=89=E9=92=AE=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index ff4bc78d..bfa04f00 100644 --- a/main.py +++ b/main.py @@ -159,7 +159,7 @@ def main(): # 整理反复出现的控件句柄组合 input_combo = [cookies, max_length_sl, md_dropdown, txt, txt2, top_p, temperature, chatbot, history, system_prompt, plugin_advanced_arg] output_combo = [cookies, chatbot, history, status] - predict_args = dict(fn=ArgsGeneralWrapper(predict), inputs=input_combo, outputs=output_combo) + predict_args = dict(fn=ArgsGeneralWrapper(predict), inputs=[*input_combo, gr.State(True)], outputs=output_combo) # 提交按钮、重置按钮 cancel_handles.append(txt.submit(**predict_args)) cancel_handles.append(txt2.submit(**predict_args)) From a1cc2f733ce3471870176e2fa6f76dcd67755c14 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Thu, 14 Sep 2023 15:26:03 +0800 Subject: [PATCH 21/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dnougat=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E9=94=81=E9=87=8A=E6=94=BEBug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/crazy_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crazy_functions/crazy_utils.py b/crazy_functions/crazy_utils.py index 6cf471e5..66c53a97 100644 --- a/crazy_functions/crazy_utils.py +++ b/crazy_functions/crazy_utils.py @@ -731,9 +731,10 @@ class nougat_interface(): from toolbox import get_log_folder, gen_time_str dst = os.path.join(get_log_folder(plugin_name='nougat'), gen_time_str()) os.makedirs(dst) - self.nougat_with_timeout(f'nougat --out "{os.path.abspath(dst)}" "{os.path.abspath(fp)}"', os.getcwd()) + self.nougat_with_timeout(f'nougat --out "{os.path.abspath(dst)}" "{os.path.abspath(fp)}"', os.getcwd(), timeout=300) res = glob.glob(os.path.join(dst,'*.mmd')) if len(res) == 0: + self.threadLock.release() raise RuntimeError("Nougat解析论文失败。") self.threadLock.release() return res[0] From a3f36668a840472e56b5ee1e2640e2a5d93f56c1 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Thu, 14 Sep 2023 17:51:41 +0800 Subject: [PATCH 22/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dlatex=E8=AF=86=E5=88=AB?= =?UTF-8?q?=E4=B8=BB=E6=96=87=E4=BB=B6=E9=94=99=E8=AF=AF=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/latex_fns/latex_toolbox.py | 1 + request_llm/com_sparkapi.py | 1 + 2 files changed, 2 insertions(+) diff --git a/crazy_functions/latex_fns/latex_toolbox.py b/crazy_functions/latex_fns/latex_toolbox.py index e39ac2db..9fc1f6c5 100644 --- a/crazy_functions/latex_fns/latex_toolbox.py +++ b/crazy_functions/latex_fns/latex_toolbox.py @@ -256,6 +256,7 @@ def find_main_tex_file(file_manifest, mode): canidates_score.append(0) with open(texf, 'r', encoding='utf8', errors='ignore') as f: file_content = f.read() + file_content = rm_comments(file_content) for uw in unexpected_words: if uw in file_content: canidates_score[-1] -= 1 diff --git a/request_llm/com_sparkapi.py b/request_llm/com_sparkapi.py index 0b8d655d..ae970b9a 100644 --- a/request_llm/com_sparkapi.py +++ b/request_llm/com_sparkapi.py @@ -109,6 +109,7 @@ class SparkRequestInstance(): code = data['header']['code'] if code != 0: print(f'请求错误: {code}, {data}') + self.result_buf += str(data) ws.close() self.time_to_exit_event.set() else: From 567c6530d844fc9e3efa7ed800c16714d5423576 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Thu, 14 Sep 2023 21:38:47 +0800 Subject: [PATCH 23/29] =?UTF-8?q?=E5=A2=9E=E5=8A=A0NOUGAT=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E6=8F=90=E7=A4=BA=E5=92=8C=E9=94=99=E8=AF=AF=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/crazy_utils.py | 13 +++++++++++-- crazy_functions/批量翻译PDF文档_NOUGAT.py | 2 +- request_llm/bridge_chatgpt.py | 5 +++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/crazy_functions/crazy_utils.py b/crazy_functions/crazy_utils.py index 66c53a97..bd2ee3f0 100644 --- a/crazy_functions/crazy_utils.py +++ b/crazy_functions/crazy_utils.py @@ -1,6 +1,7 @@ from toolbox import update_ui, get_conf, trimmed_format_exc, get_log_folder import threading import os +import logging def input_clipping(inputs, history, max_token_limit): import numpy as np @@ -714,6 +715,7 @@ class nougat_interface(): def nougat_with_timeout(self, command, cwd, timeout=3600): import subprocess + logging.info('正在执行命令', command) process = subprocess.Popen(command, shell=True, cwd=cwd) try: stdout, stderr = process.communicate(timeout=timeout) @@ -725,13 +727,20 @@ class nougat_interface(): return True - def NOUGAT_parse_pdf(self, fp): + def NOUGAT_parse_pdf(self, fp, chatbot, history): + from toolbox import update_ui_lastest_msg + + yield from update_ui_lastest_msg("正在解析论文, 请稍候。进度：正在排队, 等待线程锁...", + chatbot=chatbot, history=history, delay=0) self.threadLock.acquire() import glob, threading, os from toolbox import get_log_folder, gen_time_str dst = os.path.join(get_log_folder(plugin_name='nougat'), gen_time_str()) os.makedirs(dst) - self.nougat_with_timeout(f'nougat --out "{os.path.abspath(dst)}" "{os.path.abspath(fp)}"', os.getcwd(), timeout=300) + + yield from update_ui_lastest_msg("正在解析论文, 请稍候。进度：正在加载NOUGAT... （提示：首次运行需要花费较长时间下载NOUGAT参数）", + chatbot=chatbot, history=history, delay=0) + self.nougat_with_timeout(f'nougat --out "{os.path.abspath(dst)}" "{os.path.abspath(fp)}"', os.getcwd(), timeout=3600) res = glob.glob(os.path.join(dst,'*.mmd')) if len(res) == 0: self.threadLock.release() diff --git a/crazy_functions/批量翻译PDF文档_NOUGAT.py b/crazy_functions/批量翻译PDF文档_NOUGAT.py index 0e2fb81d..2dc15f79 100644 --- a/crazy_functions/批量翻译PDF文档_NOUGAT.py +++ b/crazy_functions/批量翻译PDF文档_NOUGAT.py @@ -100,7 +100,7 @@ def 解析PDF_基于NOUGAT(file_manifest, project_folder, llm_kwargs, plugin_kwa nougat_handle = nougat_interface() for index, fp in enumerate(file_manifest): chatbot.append(["当前进度：", f"正在解析论文，请稍候。（第一次运行时，需要花费较长时间下载NOUGAT参数）"]); yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 - fpp = nougat_handle.NOUGAT_parse_pdf(fp) + fpp = yield from nougat_handle.NOUGAT_parse_pdf(fp, chatbot, history) with open(fpp, 'r', encoding='utf8') as f: article_content = f.readlines() diff --git a/request_llm/bridge_chatgpt.py b/request_llm/bridge_chatgpt.py index 6a458f20..3fe854cf 100644 --- a/request_llm/bridge_chatgpt.py +++ b/request_llm/bridge_chatgpt.py @@ -128,6 +128,7 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp yield from update_ui(chatbot=chatbot, history=history, msg="缺少api_key") # 刷新界面 return + user_input = inputs if additional_fn is not None: from core_functional import handle_core_functionality inputs, history = handle_core_functionality(additional_fn, inputs, history, chatbot) @@ -138,8 +139,8 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp yield from update_ui(chatbot=chatbot, history=history, msg="等待响应") # 刷新界面 # check mis-behavior - if is_the_upload_folder(raw_input): - chatbot[-1] = (inputs, f"[Local Message] 检测到操作错误！当您上传文档之后，需要点击“函数插件区”按钮进行处理，而不是点击“提交”按钮。") + if is_the_upload_folder(user_input): + chatbot[-1] = (inputs, f"[Local Message] 检测到操作错误！当您上传文档之后，需点击“**函数插件区**”按钮进行处理，请勿点击“提交”按钮或者“基础功能区”按钮。") yield from update_ui(chatbot=chatbot, history=history, msg="正常") # 刷新界面 time.sleep(2) From 5ba8ea27d19d1da8d165528fbad0b9ad996f4418 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Thu, 14 Sep 2023 22:33:07 +0800 Subject: [PATCH 24/29] =?UTF-8?q?=E7=94=A8logging=E5=8F=96=E4=BB=A3print?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/批量Markdown翻译.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crazy_functions/批量Markdown翻译.py b/crazy_functions/批量Markdown翻译.py index 277d942c..9485b1ec 100644 --- a/crazy_functions/批量Markdown翻译.py +++ b/crazy_functions/批量Markdown翻译.py @@ -1,4 +1,4 @@ -import glob, time, os, re +import glob, time, os, re, logging from toolbox import update_ui, trimmed_format_exc, gen_time_str, disable_auto_promotion from toolbox import CatchException, report_execption, get_log_folder from toolbox import write_history_to_file, promote_file_to_downloadzone @@ -34,7 +34,7 @@ class PaperFileGroup(): self.sp_file_contents.append(segment) self.sp_file_index.append(index) self.sp_file_tag.append(self.file_paths[index] + f".part-{j}.md") - print('Segmentation: done') + logging.info('Segmentation: done') def merge_result(self): self.file_result = ["" for _ in range(len(self.file_paths))] @@ -101,7 +101,7 @@ def 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch pfg.merge_result() pfg.write_result(language) except: - print(trimmed_format_exc()) + logging.error(trimmed_format_exc()) # <-------- 整理结果，退出 ----------> create_report_file_name = gen_time_str() + f"-chatgpt.md" @@ -121,7 +121,7 @@ def get_files_from_everything(txt, preference=''): proxies, = get_conf('proxies') # 网络的远程文件 if preference == 'Github': - print('正在从github下载资源 ...') + logging.info('正在从github下载资源 ...') if not txt.endswith('.md'): # Make a request to the GitHub API to retrieve the repository information url = txt.replace("https://github.com/", "https://api.github.com/repos/") + '/readme' From 1c4cb340cac161c45ec5aefb8a39c308c3f5cbf6 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Thu, 14 Sep 2023 22:45:45 +0800 Subject: [PATCH 25/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=BB=9E=E7=95=99?= =?UTF-8?q?=E6=96=87=E6=A1=A3=E7=9A=84=E6=8F=90=E7=A4=BABug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- toolbox.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/toolbox.py b/toolbox.py index ccc36643..1452c13a 100644 --- a/toolbox.py +++ b/toolbox.py @@ -84,7 +84,7 @@ def ArgsGeneralWrapper(f): # 正常状态 if len(args) == 0: # 插件通道 yield from f(txt_passon, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, system_prompt, request) - else: # 对话通道 + else: # 对话通道，或者基础功能通道 yield from f(txt_passon, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, system_prompt, *args) else: # 处理少数情况下的特殊插件的锁定状态 @@ -93,7 +93,8 @@ def ArgsGeneralWrapper(f): yield from f_hot_reload(txt_passon, llm_kwargs, plugin_kwargs, chatbot_with_cookie, history, system_prompt, request) # 判断一下用户是否错误地通过对话通道进入，如果是，则进行提醒 final_cookies = chatbot_with_cookie.get_cookies() - if len(args) == 0 and 'files_to_promote' in final_cookies and len(final_cookies['files_to_promote']) > 0: + # len(args) != 0 代表“提交”键对话通道，或者基础功能通道 + if len(args) != 0 and 'files_to_promote' in final_cookies and len(final_cookies['files_to_promote']) > 0: chatbot_with_cookie.append(["检测到**滞留的缓存文档**，请及时处理。", "请及时点击“**保存当前对话**”获取所有滞留文档。"]) yield from update_ui(chatbot_with_cookie, final_cookies['history'], msg="检测到被滞留的缓存文档") return decorated From 16ff5ddcdc2be805108d8fc70d7b2ec11ba8aea2 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Thu, 14 Sep 2023 23:07:12 +0800 Subject: [PATCH 26/29] =?UTF-8?q?=E7=89=88=E6=9C=AC3.52?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version b/version index 4f234c6f..897b0428 100644 --- a/version +++ b/version @@ -1,5 +1,5 @@ { - "version": 3.51, + "version": 3.52, "show_feature": true, - "new_feature": "支持插件分类和更多UI皮肤外观 <-> 支持用户使用自然语言调度各个插件(虚空终端) ！ <-> 改进UI，设计新主题 <-> 支持借助GROBID实现PDF高精度翻译 <-> 接入百度千帆平台和文心一言 <-> 接入阿里通义千问、讯飞星火、上海AI-Lab书生 <-> 优化一键升级 <-> 提高arxiv翻译速度和成功率" + "new_feature": "提高稳定性&解决多用户冲突问题 <-> 支持插件分类和更多UI皮肤外观 <-> 支持用户使用自然语言调度各个插件(虚空终端) ！ <-> 改进UI，设计新主题 <-> 支持借助GROBID实现PDF高精度翻译 <-> 接入百度千帆平台和文心一言 <-> 接入阿里通义千问、讯飞星火、上海AI-Lab书生 <-> 优化一键升级 <-> 提高arxiv翻译速度和成功率" } From abea0d07acc6190db530223376d66aaf3d0c8be1 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Fri, 15 Sep 2023 11:00:30 +0800 Subject: [PATCH 27/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dlogging=E7=9A=84Bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/crazy_utils.py | 2 +- request_llm/bridge_chatgpt.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crazy_functions/crazy_utils.py b/crazy_functions/crazy_utils.py index bd2ee3f0..ee1ab907 100644 --- a/crazy_functions/crazy_utils.py +++ b/crazy_functions/crazy_utils.py @@ -715,7 +715,7 @@ class nougat_interface(): def nougat_with_timeout(self, command, cwd, timeout=3600): import subprocess - logging.info('正在执行命令', command) + logging.info(f'正在执行命令 {command}') process = subprocess.Popen(command, shell=True, cwd=cwd) try: stdout, stderr = process.communicate(timeout=timeout) diff --git a/request_llm/bridge_chatgpt.py b/request_llm/bridge_chatgpt.py index 3fe854cf..52290d6e 100644 --- a/request_llm/bridge_chatgpt.py +++ b/request_llm/bridge_chatgpt.py @@ -205,7 +205,7 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp chunkjson = json.loads(chunk_decoded[6:]) status_text = f"finish_reason: {chunkjson['choices'][0].get('finish_reason', 'null')}" # 如果这里抛出异常，一般是文本过长，详情见get_full_error的输出 - gpt_replying_buffer = gpt_replying_buffer + json.loads(chunk_decoded[6:])['choices'][0]["delta"]["content"] + gpt_replying_buffer = gpt_replying_buffer + chunkjson['choices'][0]["delta"]["content"] history[-1] = gpt_replying_buffer chatbot[-1] = (history[-2], history[-1]) yield from update_ui(chatbot=chatbot, history=history, msg=status_text) # 刷新界面 From 9905122fc298606a9548a5a7917d5aad224e7d30 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Fri, 15 Sep 2023 12:55:41 +0800 Subject: [PATCH 28/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8DTex=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=8C=B9=E9=85=8DBUG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crazy_functions/latex_fns/latex_toolbox.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crazy_functions/latex_fns/latex_toolbox.py b/crazy_functions/latex_fns/latex_toolbox.py index 9fc1f6c5..c733c818 100644 --- a/crazy_functions/latex_fns/latex_toolbox.py +++ b/crazy_functions/latex_fns/latex_toolbox.py @@ -291,7 +291,11 @@ def find_tex_file_ignore_case(fp): import glob for f in glob.glob(dir_name+'/*.tex'): base_name_s = os.path.basename(fp) - if base_name_s.lower() == base_name.lower(): return f + base_name_f = os.path.basename(f) + if base_name_s.lower() == base_name_f.lower(): return f + # 试着加上.tex后缀试试 + if not base_name_s.endswith('.tex'): base_name_s+='.tex' + if base_name_s.lower() == base_name_f.lower(): return f return None def merge_tex_files_(project_foler, main_file, mode): @@ -302,9 +306,9 @@ def merge_tex_files_(project_foler, main_file, mode): for s in reversed([q for q in re.finditer(r"\\input\{(.*?)\}", main_file, re.M)]): f = s.group(1) fp = os.path.join(project_foler, f) - fp = find_tex_file_ignore_case(fp) - if fp: - with open(fp, 'r', encoding='utf-8', errors='replace') as fx: c = fx.read() + fp_ = find_tex_file_ignore_case(fp) + if fp_: + with open(fp_, 'r', encoding='utf-8', errors='replace') as fx: c = fx.read() else: raise RuntimeError(f'找不到{fp}，Tex源文件缺失！') c = merge_tex_files_(project_foler, c, mode) From 760ff1840ce3b9c71f2ff36f3a2bb78894946f10 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Fri, 15 Sep 2023 17:08:23 +0800 Subject: [PATCH 29/29] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=80=E4=B8=AA?= =?UTF-8?q?=E5=BE=AA=E7=8E=AF=E7=9A=84Bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request_llm/bridge_chatgpt.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/request_llm/bridge_chatgpt.py b/request_llm/bridge_chatgpt.py index 52290d6e..a1b6ba47 100644 --- a/request_llm/bridge_chatgpt.py +++ b/request_llm/bridge_chatgpt.py @@ -72,6 +72,7 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", stream_response = response.iter_lines() result = '' + json_data = None while True: try: chunk = next(stream_response).decode() except StopIteration: @@ -90,20 +91,21 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="", delta = json_data["delta"] if len(delta) == 0: break if "role" in delta: continue - if "content" in delta: + if "content" in delta: result += delta["content"] if not console_slience: print(delta["content"], end='') if observe_window is not None: # 观测窗，把已经获取的数据显示出去 - if len(observe_window) >= 1: observe_window[0] += delta["content"] + if len(observe_window) >= 1: + observe_window[0] += delta["content"] # 看门狗，如果超过期限没有喂狗，则终止 - if len(observe_window) >= 2: + if len(observe_window) >= 2: if (time.time()-observe_window[1]) > watch_dog_patience: raise RuntimeError("用户取消了程序。") else: raise RuntimeError("意外Json结构："+delta) - if json_data['finish_reason'] == 'content_filter': + if json_data and json_data['finish_reason'] == 'content_filter': raise RuntimeError("由于提问含不合规内容被Azure过滤。") - if json_data['finish_reason'] == 'length': + if json_data and json_data['finish_reason'] == 'length': raise ConnectionAbortedError("正常结束，但显示Token不足，导致输出不完整，请削减单次输入的文本量。") return result