diff --git a/.github/workflows/build-with-all-capacity.yml b/.github/workflows/build-with-all-capacity.yml
new file mode 100644
index 00000000..7b2ee6ae
--- /dev/null
+++ b/.github/workflows/build-with-all-capacity.yml
@@ -0,0 +1,44 @@
+# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images#publishing-images-to-github-packages
+name: build-with-all-capacity
+
+on:
+ push:
+ branches:
+ - 'master'
+
+env:
+ REGISTRY: ghcr.io
+ IMAGE_NAME: ${{ github.repository }}_with_all_capacity
+
+jobs:
+ build-and-push-image:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: write
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Log in to the Container registry
+ uses: docker/login-action@v2
+ with:
+ registry: ${{ env.REGISTRY }}
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Extract metadata (tags, labels) for Docker
+ id: meta
+ uses: docker/metadata-action@v4
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+ - name: Build and push Docker image
+ uses: docker/build-push-action@v4
+ with:
+ context: .
+ push: true
+ file: docs/GithubAction+AllCapacity
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
new file mode 100644
index 00000000..717f2543
--- /dev/null
+++ b/.github/workflows/stale.yml
@@ -0,0 +1,25 @@
+# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
+#
+# You can adjust the behavior by modifying this file.
+# For more information, see:
+# https://github.com/actions/stale
+
+name: 'Close stale issues and PRs'
+on:
+ schedule:
+ - cron: '*/5 * * * *'
+
+jobs:
+ stale:
+ runs-on: ubuntu-latest
+ permissions:
+ issues: write
+ pull-requests: read
+
+ steps:
+ - uses: actions/stale@v8
+ with:
+ stale-issue-message: 'This issue is stale because it has been open 100 days with no activity. Remove stale label or comment or this will be closed in 1 days.'
+ days-before-stale: 100
+ days-before-close: 1
+ debug-only: true
diff --git a/README.md b/README.md
index 00017df5..6d8e3eb6 100644
--- a/README.md
+++ b/README.md
@@ -22,13 +22,13 @@ pinned: false
**如果喜欢这个项目,请给它一个Star;如果您发明了好用的快捷键或函数插件,欢迎发pull requests!**
If you like this project, please give it a Star. If you've come up with more useful academic shortcuts or functional plugins, feel free to open an issue or pull request. We also have a README in [English|](docs/README_EN.md)[日本語|](docs/README_JP.md)[한국어|](https://github.com/mldljyh/ko_gpt_academic)[Русский|](docs/README_RS.md)[Français](docs/README_FR.md) translated by this project itself.
-To translate this project to arbitary language with GPT, read and run [`multi_language.py`](multi_language.py) (experimental).
+To translate this project to arbitrary language with GPT, read and run [`multi_language.py`](multi_language.py) (experimental).
> **Note**
>
-> 1.请注意只有 **高亮(如红色)** 标识的函数插件(按钮)才支持读取文件,部分插件位于插件区的**下拉菜单**中。另外我们以**最高优先级**欢迎和处理任何新插件的PR。
+> 1.请注意只有 **高亮** 标识的函数插件(按钮)才支持读取文件,部分插件位于插件区的**下拉菜单**中。另外我们以**最高优先级**欢迎和处理任何新插件的PR。
>
-> 2.本项目中每个文件的功能都在自译解[`self_analysis.md`](https://github.com/binary-husky/gpt_academic/wiki/chatgpt-academic%E9%A1%B9%E7%9B%AE%E8%87%AA%E8%AF%91%E8%A7%A3%E6%8A%A5%E5%91%8A)详细说明。随着版本的迭代,您也可以随时自行点击相关函数插件,调用GPT重新生成项目的自我解析报告。常见问题汇总在[`wiki`](https://github.com/binary-husky/gpt_academic/wiki/%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98)当中。[安装方法](#installation)。
+> 2.本项目中每个文件的功能都在[自译解报告`self_analysis.md`](https://github.com/binary-husky/gpt_academic/wiki/GPT‐Academic项目自译解报告)详细说明。随着版本的迭代,您也可以随时自行点击相关函数插件,调用GPT重新生成项目的自我解析报告。常见问题[`wiki`](https://github.com/binary-husky/gpt_academic/wiki)。[安装方法](#installation) | [配置说明](https://github.com/binary-husky/gpt_academic/wiki/%E9%A1%B9%E7%9B%AE%E9%85%8D%E7%BD%AE%E8%AF%B4%E6%98%8E)。
>
> 3.本项目兼容并鼓励尝试国产大语言模型ChatGLM和Moss等等。支持多个api-key共存,可在配置文件中填写如`API_KEY="openai-key1,openai-key2,azure-key3,api2d-key4"`。需要临时更换`API_KEY`时,在输入区输入临时的`API_KEY`然后回车键提交后即可生效。
@@ -65,7 +65,8 @@ Latex论文一键校对 | [函数插件] 仿Grammarly对Latex文章进行语法
[多LLM模型](https://www.bilibili.com/video/BV1wT411p7yf)支持 | 同时被GPT3.5、GPT4、[清华ChatGLM2](https://github.com/THUDM/ChatGLM2-6B)、[复旦MOSS](https://github.com/OpenLMLab/MOSS)同时伺候的感觉一定会很不错吧?
⭐ChatGLM2微调模型 | 支持加载ChatGLM2微调模型,提供ChatGLM2微调辅助插件
更多LLM模型接入,支持[huggingface部署](https://huggingface.co/spaces/qingxu98/gpt-academic) | 加入Newbing接口(新必应),引入清华[Jittorllms](https://github.com/Jittor/JittorLLMs)支持[LLaMA](https://github.com/facebookresearch/llama)和[盘古α](https://openi.org.cn/pangu/)
-⭐[虚空终端](https://github.com/binary-husky/void-terminal)pip包 | 脱离GUI,在Python中直接调用本项目的函数插件(开发中)
+⭐[void-terminal](https://github.com/binary-husky/void-terminal) pip包 | 脱离GUI,在Python中直接调用本项目的所有函数插件(开发中)
+⭐虚空终端插件 | [函数插件] 用自然语言,直接调度本项目其他插件
更多新功能展示 (图像生成等) …… | 见本文档结尾处 ……
@@ -114,7 +115,7 @@ cd gpt_academic
在`config.py`中,配置API KEY等设置,[点击查看特殊网络环境设置方法](https://github.com/binary-husky/gpt_academic/issues/1) 。
-(P.S. 程序运行时会优先检查是否存在名为`config_private.py`的私密配置文件,并用其中的配置覆盖`config.py`的同名配置。因此,如果您能理解我们的配置读取逻辑,我们强烈建议您在`config.py`旁边创建一个名为`config_private.py`的新配置文件,并把`config.py`中的配置转移(复制)到`config_private.py`中。`config_private.py`不受git管控,可以让您的隐私信息更加安全。P.S.项目同样支持通过`环境变量`配置大多数选项,环境变量的书写格式参考`docker-compose`文件。读取优先级: `环境变量` > `config_private.py` > `config.py`)
+(P.S. 程序运行时会优先检查是否存在名为`config_private.py`的私密配置文件,并用其中的配置覆盖`config.py`的同名配置。因此,如果您能理解我们的配置读取逻辑,我们强烈建议您在`config.py`旁边创建一个名为`config_private.py`的新配置文件,并把`config.py`中的配置转移(复制)到`config_private.py`中(仅复制您修改过的配置条目即可)。`config_private.py`不受git管控,可以让您的隐私信息更加安全。P.S.项目同样支持通过`环境变量`配置大多数选项,环境变量的书写格式参考`docker-compose`文件。读取优先级: `环境变量` > `config_private.py` > `config.py`)
3. 安装依赖
@@ -160,11 +161,14 @@ python main.py
### 安装方法II:使用Docker
+[](https://github.com/binary-husky/gpt_academic/actions/workflows/build-with-audio-assistant.yml)
+
1. 仅ChatGPT(推荐大多数人选择,等价于docker-compose方案1)
[](https://github.com/binary-husky/gpt_academic/actions/workflows/build-without-local-llms.yml)
[](https://github.com/binary-husky/gpt_academic/actions/workflows/build-with-latex.yml)
[](https://github.com/binary-husky/gpt_academic/actions/workflows/build-with-audio-assistant.yml)
+
``` sh
git clone --depth=1 https://github.com/binary-husky/gpt_academic.git # 下载项目
cd gpt_academic # 进入路径
@@ -261,10 +265,13 @@ Tip:不指定文件直接点击 `载入对话历史存档` 可以查看历史h
-3. 生成报告。大部分插件都会在执行结束后,生成工作报告
+3. 虚空终端(从自然语言输入中,理解用户意图+自动调用其他插件)
+
+- 步骤一:输入 “ 请调用插件翻译PDF论文,地址为https://storage.googleapis.com/deepmind-media/alphago/AlphaGoNaturePaper.pdf ”
+- 步骤二:点击“虚空终端”
+
4. 模块化功能设计,简单的接口却能支持强大的功能
@@ -311,8 +318,10 @@ Tip:不指定文件直接点击 `载入对话历史存档` 可以查看历史h
+
### II:版本:
-- version 3.5(Todo): 使用自然语言调用本项目的所有函数插件(高优先级)
+- version 3.60(todo): 优化虚空终端,引入code interpreter和更多插件
+- version 3.50: 使用自然语言调用本项目的所有函数插件(虚空终端),支持插件分类,改进UI,设计新主题
- version 3.49: 支持百度千帆平台和文心一言
- version 3.48: 支持阿里达摩院通义千问,上海AI-Lab书生,讯飞星火
- version 3.46: 支持完全脱手操作的实时语音对话
diff --git a/app.py b/app.py
index 3463fb6b..2da18793 100644
--- a/app.py
+++ b/app.py
@@ -7,18 +7,18 @@ def main():
from request_llm.bridge_all import predict
from toolbox import format_io, find_free_port, on_file_uploaded, on_report_generated, get_conf, ArgsGeneralWrapper, load_chat_cookies, DummyWith
# 建议您复制一个config_private.py放自己的秘密, 如API和代理网址, 避免不小心传github被别人看到
- proxies, WEB_PORT, LLM_MODEL, CONCURRENT_COUNT, AUTHENTICATION, CHATBOT_HEIGHT, LAYOUT, AVAIL_LLM_MODELS, AUTO_CLEAR_TXT = \
- get_conf('proxies', 'WEB_PORT', 'LLM_MODEL', 'CONCURRENT_COUNT', 'AUTHENTICATION', 'CHATBOT_HEIGHT', 'LAYOUT', 'AVAIL_LLM_MODELS', 'AUTO_CLEAR_TXT')
+ proxies, WEB_PORT, LLM_MODEL, CONCURRENT_COUNT, AUTHENTICATION = get_conf('proxies', 'WEB_PORT', 'LLM_MODEL', 'CONCURRENT_COUNT', 'AUTHENTICATION')
+ CHATBOT_HEIGHT, LAYOUT, AVAIL_LLM_MODELS, AUTO_CLEAR_TXT = get_conf('CHATBOT_HEIGHT', 'LAYOUT', 'AVAIL_LLM_MODELS', 'AUTO_CLEAR_TXT')
ENABLE_AUDIO, AUTO_CLEAR_TXT = get_conf('ENABLE_AUDIO', 'AUTO_CLEAR_TXT')
+
# 如果WEB_PORT是-1, 则随机选取WEB端口
PORT = find_free_port() if WEB_PORT <= 0 else WEB_PORT
- if not AUTHENTICATION: AUTHENTICATION = None
-
from check_proxy import get_current_version
from themes.theme import adjust_theme, advanced_css, theme_declaration
initial_prompt = "Serve me as a writing and programming assistant."
title_html = f"GPT 学术优化 {get_current_version()}
{theme_declaration}"
- description = """代码开源和更新[地址🚀](https://github.com/binary-husky/chatgpt_academic),感谢热情的[开发者们❤️](https://github.com/binary-husky/chatgpt_academic/graphs/contributors)"""
+ description = "代码开源和更新[地址🚀](https://github.com/binary-husky/gpt_academic),"
+ description += "感谢热情的[开发者们❤️](https://github.com/binary-husky/gpt_academic/graphs/contributors)"
# 问询记录, python 版本建议3.9+(越新越好)
import logging, uuid
@@ -35,7 +35,10 @@ def main():
# 高级函数插件
from crazy_functional import get_crazy_functions
- crazy_fns = get_crazy_functions()
+ DEFAULT_FN_GROUPS, = get_conf('DEFAULT_FN_GROUPS')
+ plugins = get_crazy_functions()
+ all_plugin_groups = list(set([g for _, plugin in plugins.items() for g in plugin['Group'].split('|')]))
+ match_group = lambda tags, groups: any([g in groups for g in tags.split('|')])
# 处理markdown文本格式的转变
gr.Chatbot.postprocess = format_io
@@ -85,25 +88,33 @@ def main():
if ("Visible" in functional[k]) and (not functional[k]["Visible"]): continue
variant = functional[k]["Color"] if "Color" in functional[k] else "secondary"
functional[k]["Button"] = gr.Button(k, variant=variant)
+ functional[k]["Button"].style(size="sm")
with gr.Accordion("函数插件区", open=True, elem_id="plugin-panel") as area_crazy_fn:
with gr.Row():
gr.Markdown("插件可读取“输入区”文本/路径作为参数(上传文件自动修正路径)")
+ with gr.Row(elem_id="input-plugin-group"):
+ plugin_group_sel = gr.Dropdown(choices=all_plugin_groups, label='', show_label=False, value=DEFAULT_FN_GROUPS,
+ multiselect=True, interactive=True, elem_classes='normal_mut_select').style(container=False)
with gr.Row():
- for k in crazy_fns:
- if not crazy_fns[k].get("AsButton", True): continue
- variant = crazy_fns[k]["Color"] if "Color" in crazy_fns[k] else "secondary"
- crazy_fns[k]["Button"] = gr.Button(k, variant=variant)
- crazy_fns[k]["Button"].style(size="sm")
+ for k, plugin in plugins.items():
+ if not plugin.get("AsButton", True): continue
+ visible = True if match_group(plugin['Group'], DEFAULT_FN_GROUPS) else False
+ variant = plugins[k]["Color"] if "Color" in plugin else "secondary"
+ plugin['Button'] = plugins[k]['Button'] = gr.Button(k, variant=variant, visible=visible).style(size="sm")
with gr.Row():
with gr.Accordion("更多函数插件", open=True):
- dropdown_fn_list = [k for k in crazy_fns.keys() if not crazy_fns[k].get("AsButton", True)]
+ dropdown_fn_list = []
+ for k, plugin in plugins.items():
+ if not match_group(plugin['Group'], DEFAULT_FN_GROUPS): continue
+ if not plugin.get("AsButton", True): dropdown_fn_list.append(k) # 排除已经是按钮的插件
+ elif plugin.get('AdvancedArgs', False): dropdown_fn_list.append(k) # 对于需要高级参数的插件,亦在下拉菜单中显示
with gr.Row():
dropdown = gr.Dropdown(dropdown_fn_list, value=r"打开插件列表", label="", show_label=False).style(container=False)
with gr.Row():
plugin_advanced_arg = gr.Textbox(show_label=True, label="高级参数输入区", visible=False,
placeholder="这里是特殊函数插件的高级参数输入区").style(container=False)
with gr.Row():
- switchy_bt = gr.Button(r"请先从插件列表中选择", variant="secondary")
+ switchy_bt = gr.Button(r"请先从插件列表中选择", variant="secondary").style(size="sm")
with gr.Row():
with gr.Accordion("点击展开“文件上传区”。上传本地文件/压缩包供函数插件调用。", open=False) as area_file_up:
file_upload = gr.Files(label="任何文件, 但推荐上传压缩文件(zip, tar)", file_count="multiple")
@@ -114,7 +125,6 @@ def main():
max_length_sl = gr.Slider(minimum=256, maximum=8192, value=4096, step=1, interactive=True, label="Local LLM MaxLength",)
checkboxes = gr.CheckboxGroup(["基础功能区", "函数插件区", "底部输入区", "输入清除键", "插件参数区"], value=["基础功能区", "函数插件区"], label="显示/隐藏功能区")
md_dropdown = gr.Dropdown(AVAIL_LLM_MODELS, value=LLM_MODEL, label="更换LLM模型/请求源").style(container=False)
-
gr.Markdown(description)
with gr.Accordion("备选输入区", open=True, visible=False, elem_id="input-panel2") as area_input_secondary:
with gr.Row():
@@ -125,6 +135,7 @@ def main():
resetBtn2 = gr.Button("重置", variant="secondary"); resetBtn2.style(size="sm")
stopBtn2 = gr.Button("停止", variant="secondary"); stopBtn2.style(size="sm")
clearBtn2 = gr.Button("清除", variant="secondary", visible=False); clearBtn2.style(size="sm")
+
# 功能区显示开关与功能区的互动
def fn_area_visibility(a):
ret = {}
@@ -162,19 +173,19 @@ def main():
click_handle = functional[k]["Button"].click(fn=ArgsGeneralWrapper(predict), inputs=[*input_combo, gr.State(True), gr.State(k)], outputs=output_combo)
cancel_handles.append(click_handle)
# 文件上传区,接收文件后与chatbot的互动
- file_upload.upload(on_file_uploaded, [file_upload, chatbot, txt, txt2, checkboxes], [chatbot, txt, txt2])
+ file_upload.upload(on_file_uploaded, [file_upload, chatbot, txt, txt2, checkboxes, cookies], [chatbot, txt, txt2, cookies])
# 函数插件-固定按钮区
- for k in crazy_fns:
- if not crazy_fns[k].get("AsButton", True): continue
- click_handle = crazy_fns[k]["Button"].click(ArgsGeneralWrapper(crazy_fns[k]["Function"]), [*input_combo, gr.State(PORT)], output_combo)
+ for k in plugins:
+ if not plugins[k].get("AsButton", True): continue
+ click_handle = plugins[k]["Button"].click(ArgsGeneralWrapper(plugins[k]["Function"]), [*input_combo, gr.State(PORT)], output_combo)
click_handle.then(on_report_generated, [cookies, file_upload, chatbot], [cookies, file_upload, chatbot])
cancel_handles.append(click_handle)
# 函数插件-下拉菜单与随变按钮的互动
def on_dropdown_changed(k):
- variant = crazy_fns[k]["Color"] if "Color" in crazy_fns[k] else "secondary"
+ variant = plugins[k]["Color"] if "Color" in plugins[k] else "secondary"
ret = {switchy_bt: gr.update(value=k, variant=variant)}
- if crazy_fns[k].get("AdvancedArgs", False): # 是否唤起高级插件参数区
- ret.update({plugin_advanced_arg: gr.update(visible=True, label=f"插件[{k}]的高级参数说明:" + crazy_fns[k].get("ArgsReminder", [f"没有提供高级参数功能说明"]))})
+ if plugins[k].get("AdvancedArgs", False): # 是否唤起高级插件参数区
+ ret.update({plugin_advanced_arg: gr.update(visible=True, label=f"插件[{k}]的高级参数说明:" + plugins[k].get("ArgsReminder", [f"没有提供高级参数功能说明"]))})
else:
ret.update({plugin_advanced_arg: gr.update(visible=False, label=f"插件[{k}]不需要高级参数。")})
return ret
@@ -185,13 +196,26 @@ def main():
# 随变按钮的回调函数注册
def route(request: gr.Request, k, *args, **kwargs):
if k in [r"打开插件列表", r"请先从插件列表中选择"]: return
- yield from ArgsGeneralWrapper(crazy_fns[k]["Function"])(request, *args, **kwargs)
+ yield from ArgsGeneralWrapper(plugins[k]["Function"])(request, *args, **kwargs)
click_handle = switchy_bt.click(route,[switchy_bt, *input_combo, gr.State(PORT)], output_combo)
click_handle.then(on_report_generated, [cookies, file_upload, chatbot], [cookies, file_upload, chatbot])
cancel_handles.append(click_handle)
# 终止按钮的回调函数注册
stopBtn.click(fn=None, inputs=None, outputs=None, cancels=cancel_handles)
stopBtn2.click(fn=None, inputs=None, outputs=None, cancels=cancel_handles)
+ plugins_as_btn = {name:plugin for name, plugin in plugins.items() if plugin.get('Button', None)}
+ def on_group_change(group_list):
+ btn_list = []
+ fns_list = []
+ if not group_list: # 处理特殊情况:没有选择任何插件组
+ return [*[plugin['Button'].update(visible=False) for _, plugin in plugins_as_btn.items()], gr.Dropdown.update(choices=[])]
+ for k, plugin in plugins.items():
+ if plugin.get("AsButton", True):
+ btn_list.append(plugin['Button'].update(visible=match_group(plugin['Group'], group_list))) # 刷新按钮
+ if plugin.get('AdvancedArgs', False): dropdown_fn_list.append(k) # 对于需要高级参数的插件,亦在下拉菜单中显示
+ elif match_group(plugin['Group'], group_list): fns_list.append(k) # 刷新下拉列表
+ return [*btn_list, gr.Dropdown.update(choices=fns_list)]
+ plugin_group_sel.select(fn=on_group_change, inputs=[plugin_group_sel], outputs=[*[plugin['Button'] for name, plugin in plugins_as_btn.items()], dropdown])
if ENABLE_AUDIO:
from crazy_functions.live_audio.audio_io import RealtimeAudioDistribution
rad = RealtimeAudioDistribution()
diff --git a/check_proxy.py b/check_proxy.py
index 474988c1..b6fe99f8 100644
--- a/check_proxy.py
+++ b/check_proxy.py
@@ -5,7 +5,7 @@ def check_proxy(proxies):
try:
response = requests.get("https://ipapi.co/json/", proxies=proxies, timeout=4)
data = response.json()
- print(f'查询代理的地理位置,返回的结果是{data}')
+ # print(f'查询代理的地理位置,返回的结果是{data}')
if 'country_name' in data:
country = data['country_name']
result = f"代理配置 {proxies_https}, 代理所在地:{country}"
diff --git a/config.py b/config.py
index a161bb19..4225a443 100644
--- a/config.py
+++ b/config.py
@@ -47,7 +47,11 @@ API_URL_REDIRECT = {}
DEFAULT_WORKER_NUM = 3
-# 对话窗的高度
+# 色彩主题,可选 ["Default", "Chuanhu-Small-and-Beautiful", "High-Contrast"]
+THEME = "Default"
+
+
+# 对话窗的高度 (仅在LAYOUT="TOP-DOWN"时生效)
CHATBOT_HEIGHT = 1115
@@ -75,8 +79,26 @@ MAX_RETRY = 2
LLM_MODEL = "gpt-3.5-turbo" # 可选 "chatglm"
AVAIL_LLM_MODELS = ["gpt-3.5-turbo", "gpt-4", "api2d-gpt-4", "api2d-gpt-3.5-turbo", "spark", "azure-gpt-3.5"]
-# ChatGLM(2) Finetune Model Path (如果使用ChatGLM2微调模型,需要把"chatglmft"加入AVAIL_LLM_MODELS中)
-ChatGLM_PTUNING_CHECKPOINT = "" # 例如"/home/hmp/ChatGLM2-6B/ptuning/output/6b-pt-128-1e-2/checkpoint-100"
+# 插件分类默认选项
+DEFAULT_FN_GROUPS = ['对话', '编程', '学术']
+
+
+# 模型选择是 (注意: LLM_MODEL是默认选中的模型, 它*必须*被包含在AVAIL_LLM_MODELS列表中 )
+LLM_MODEL = "gpt-3.5-turbo" # 可选 ↓↓↓
+AVAIL_LLM_MODELS = ["gpt-3.5-turbo-16k", "gpt-3.5-turbo", "azure-gpt-3.5", "api2d-gpt-3.5-turbo",
+ "gpt-4", "api2d-gpt-4", "chatglm", "moss", "newbing", "stack-claude"]
+# P.S. 其他可用的模型还包括 ["qianfan", "llama2", "qwen", "gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k-0613",
+# "spark", "sparkv2", "chatglm_onnx", "claude-1-100k", "claude-2", "internlm", "jittorllms_pangualpha", "jittorllms_llama"]
+
+
+# 百度千帆(LLM_MODEL="qianfan")
+BAIDU_CLOUD_API_KEY = ''
+BAIDU_CLOUD_SECRET_KEY = ''
+BAIDU_CLOUD_QIANFAN_MODEL = 'ERNIE-Bot' # 可选 "ERNIE-Bot"(文心一言), "ERNIE-Bot-turbo", "BLOOMZ-7B", "Llama-2-70B-Chat", "Llama-2-13B-Chat", "Llama-2-7B-Chat"
+
+
+# 如果使用ChatGLM2微调模型,请把 LLM_MODEL="chatglmft",并在此处指定模型路径
+CHATGLM_PTUNING_CHECKPOINT = "" # 例如"/home/hmp/ChatGLM2-6B/ptuning/output/6b-pt-128-1e-2/checkpoint-100"
# 本地LLM模型如ChatGLM的执行方式 CPU/GPU
@@ -92,10 +114,6 @@ CONCURRENT_COUNT = 100
AUTO_CLEAR_TXT = False
-# 色彩主体,可选 ["Default", "Chuanhu-Small-and-Beautiful"]
-THEME = "Default"
-
-
# 加一个live2d装饰
ADD_WAIFU = False
@@ -161,10 +179,13 @@ HUGGINGFACE_ACCESS_TOKEN = "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"
# 获取方法:复制以下空间https://huggingface.co/spaces/qingxu98/grobid,设为public,然后GROBID_URL = "https://(你的hf用户名如qingxu98)-(你的填写的空间名如grobid).hf.space"
GROBID_URLS = [
"https://qingxu98-grobid.hf.space","https://qingxu98-grobid2.hf.space","https://qingxu98-grobid3.hf.space",
- "https://shaocongma-grobid.hf.space","https://FBR123-grobid.hf.space",
+ "https://shaocongma-grobid.hf.space","https://FBR123-grobid.hf.space", "https://yeku-grobid.hf.space",
]
+# 是否允许通过自然语言描述修改本页的配置,该功能具有一定的危险性,默认关闭
+ALLOW_RESET_CONFIG = False
+
"""
在线大模型配置关联关系示意图
@@ -182,7 +203,7 @@ GROBID_URLS = [
│ ├── AZURE_ENGINE
│ └── API_URL_REDIRECT
│
-├── "spark" 星火认知大模型
+├── "spark" 星火认知大模型 spark & sparkv2
│ ├── XFYUN_APPID
│ ├── XFYUN_API_SECRET
│ └── XFYUN_API_KEY
@@ -203,6 +224,18 @@ GROBID_URLS = [
├── NEWBING_STYLE
└── NEWBING_COOKIES
+
+用户图形界面布局依赖关系示意图
+│
+├── CHATBOT_HEIGHT 对话窗的高度
+├── CODE_HIGHLIGHT 代码高亮
+├── LAYOUT 窗口布局
+├── DARK_MODE 暗色模式 / 亮色模式
+├── DEFAULT_FN_GROUPS 插件分类默认选项
+├── THEME 色彩主题
+├── AUTO_CLEAR_TXT 是否在提交时自动清空输入框
+├── ADD_WAIFU 加一个live2d装饰
+├── ALLOW_RESET_CONFIG 是否允许通过自然语言描述修改本页的配置,该功能具有一定的危险性
插件在线服务配置依赖关系示意图
diff --git a/core_functional.py b/core_functional.py
index b04e1e0d..c4519ef8 100644
--- a/core_functional.py
+++ b/core_functional.py
@@ -63,6 +63,7 @@ def get_core_functions():
"英译中": {
"Prefix": r"翻译成地道的中文:" + "\n\n",
"Suffix": r"",
+ "Visible": False,
},
"找图片": {
"Prefix": r"我需要你找一张网络图片。使用Unsplash API(https://source.unsplash.com/960x640/?<英语关键词>)获取图片URL," +
@@ -78,6 +79,7 @@ def get_core_functions():
"Prefix": r"Here are some bibliography items, please transform them into bibtex style." +
r"Note that, reference styles maybe more than one kind, you should transform each item correctly." +
r"Items need to be transformed:",
+ "Visible": False,
"Suffix": r"",
}
}
diff --git a/crazy_functional.py b/crazy_functional.py
index 119ff0cd..c6578554 100644
--- a/crazy_functional.py
+++ b/crazy_functional.py
@@ -2,7 +2,6 @@ from toolbox import HotReload # HotReload 的意思是热更新,修改函数
def get_crazy_functions():
- ###################### 第一组插件 ###########################
from crazy_functions.读文章写摘要 import 读文章写摘要
from crazy_functions.生成函数注释 import 批量生成函数注释
from crazy_functions.解析项目源代码 import 解析项目本身
@@ -25,114 +24,8 @@ def get_crazy_functions():
from crazy_functions.对话历史存档 import 载入对话历史存档
from crazy_functions.对话历史存档 import 删除所有本地对话历史记录
from crazy_functions.辅助功能 import 清除缓存
-
from crazy_functions.批量Markdown翻译 import Markdown英译中
- function_plugins = {
- "解析整个Python项目": {
- "Color": "stop", # 按钮颜色
- "Function": HotReload(解析一个Python项目)
- },
- "载入对话历史存档(先上传存档或输入路径)": {
- "Color": "stop",
- "AsButton":False,
- "Function": HotReload(载入对话历史存档)
- },
- "删除所有本地对话历史记录(请谨慎操作)": {
- "AsButton":False,
- "Function": HotReload(删除所有本地对话历史记录)
- },
- "清除所有缓存文件(请谨慎操作)": {
- "Color": "stop",
- "AsButton": False, # 加入下拉菜单中
- "Function": HotReload(清除缓存)
- },
- "解析Jupyter Notebook文件": {
- "Color": "stop",
- "AsButton":False,
- "Function": HotReload(解析ipynb文件),
- "AdvancedArgs": True, # 调用时,唤起高级参数输入区(默认False)
- "ArgsReminder": "若输入0,则不解析notebook中的Markdown块", # 高级参数输入区的显示提示
- },
- "批量总结Word文档": {
- "Color": "stop",
- "Function": HotReload(总结word文档)
- },
- "解析整个C++项目头文件": {
- "Color": "stop", # 按钮颜色
- "AsButton": False, # 加入下拉菜单中
- "Function": HotReload(解析一个C项目的头文件)
- },
- "解析整个C++项目(.cpp/.hpp/.c/.h)": {
- "Color": "stop", # 按钮颜色
- "AsButton": False, # 加入下拉菜单中
- "Function": HotReload(解析一个C项目)
- },
- "解析整个Go项目": {
- "Color": "stop", # 按钮颜色
- "AsButton": False, # 加入下拉菜单中
- "Function": HotReload(解析一个Golang项目)
- },
- "解析整个Rust项目": {
- "Color": "stop", # 按钮颜色
- "AsButton": False, # 加入下拉菜单中
- "Function": HotReload(解析一个Rust项目)
- },
- "解析整个Java项目": {
- "Color": "stop", # 按钮颜色
- "AsButton": False, # 加入下拉菜单中
- "Function": HotReload(解析一个Java项目)
- },
- "解析整个前端项目(js,ts,css等)": {
- "Color": "stop", # 按钮颜色
- "AsButton": False, # 加入下拉菜单中
- "Function": HotReload(解析一个前端项目)
- },
- "解析整个Lua项目": {
- "Color": "stop", # 按钮颜色
- "AsButton": False, # 加入下拉菜单中
- "Function": HotReload(解析一个Lua项目)
- },
- "解析整个CSharp项目": {
- "Color": "stop", # 按钮颜色
- "AsButton": False, # 加入下拉菜单中
- "Function": HotReload(解析一个CSharp项目)
- },
- "读Tex论文写摘要": {
- "Color": "stop", # 按钮颜色
- "Function": HotReload(读文章写摘要)
- },
- "Markdown/Readme英译中": {
- # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效
- "Color": "stop",
- "Function": HotReload(Markdown英译中)
- },
- "批量生成函数注释": {
- "Color": "stop", # 按钮颜色
- "AsButton": False, # 加入下拉菜单中
- "Function": HotReload(批量生成函数注释)
- },
- "保存当前的对话": {
- "Function": HotReload(对话历史存档)
- },
- "[多线程Demo] 解析此项目本身(源码自译解)": {
- "AsButton": False, # 加入下拉菜单中
- "Function": HotReload(解析项目本身)
- },
- # "[老旧的Demo] 把本项目源代码切换成全英文": {
- # # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效
- # "AsButton": False, # 加入下拉菜单中
- # "Function": HotReload(全项目切换英文)
- # },
- "[插件demo] 历史上的今天": {
- # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效
- "Function": HotReload(高阶功能模板函数)
- },
-
- }
- ###################### 第二组插件 ###########################
- # [第二组插件]: 经过充分测试
from crazy_functions.批量总结PDF文档 import 批量总结PDF文档
- # from crazy_functions.批量总结PDF文档pdfminer import 批量总结PDF文档pdfminer
from crazy_functions.批量翻译PDF文档_多线程 import 批量翻译PDF文档
from crazy_functions.谷歌检索小助手 import 谷歌检索小助手
from crazy_functions.理解PDF文档内容 import 理解PDF文档内容标准文件输入
@@ -141,88 +34,248 @@ def get_crazy_functions():
from crazy_functions.Latex全文翻译 import Latex中译英
from crazy_functions.Latex全文翻译 import Latex英译中
from crazy_functions.批量Markdown翻译 import Markdown中译英
+ from crazy_functions.虚空终端 import 虚空终端
- function_plugins.update({
- "批量翻译PDF文档(多线程)": {
+
+ function_plugins = {
+ "虚空终端": {
+ "Group": "对话|编程|学术",
"Color": "stop",
- "AsButton": True, # 加入下拉菜单中
+ "AsButton": True,
+ "Function": HotReload(虚空终端)
+ },
+ "解析整个Python项目": {
+ "Group": "编程",
+ "Color": "stop",
+ "AsButton": True,
+ "Info": "解析一个Python项目的所有源文件(.py) | 输入参数为路径",
+ "Function": HotReload(解析一个Python项目)
+ },
+ "载入对话历史存档(先上传存档或输入路径)": {
+ "Group": "对话",
+ "Color": "stop",
+ "AsButton": False,
+ "Info": "载入对话历史存档 | 输入参数为路径",
+ "Function": HotReload(载入对话历史存档)
+ },
+ "删除所有本地对话历史记录(谨慎操作)": {
+ "Group": "对话",
+ "AsButton": False,
+ "Info": "删除所有本地对话历史记录,谨慎操作 | 不需要输入参数",
+ "Function": HotReload(删除所有本地对话历史记录)
+ },
+ "清除所有缓存文件(谨慎操作)": {
+ "Group": "对话",
+ "Color": "stop",
+ "AsButton": False, # 加入下拉菜单中
+ "Info": "清除所有缓存文件,谨慎操作 | 不需要输入参数",
+ "Function": HotReload(清除缓存)
+ },
+ "批量总结Word文档": {
+ "Group": "学术",
+ "Color": "stop",
+ "AsButton": True,
+ "Info": "批量总结word文档 | 输入参数为路径",
+ "Function": HotReload(总结word文档)
+ },
+ "解析整个C++项目头文件": {
+ "Group": "编程",
+ "Color": "stop",
+ "AsButton": False, # 加入下拉菜单中
+ "Info": "解析一个C++项目的所有头文件(.h/.hpp) | 输入参数为路径",
+ "Function": HotReload(解析一个C项目的头文件)
+ },
+ "解析整个C++项目(.cpp/.hpp/.c/.h)": {
+ "Group": "编程",
+ "Color": "stop",
+ "AsButton": False, # 加入下拉菜单中
+ "Info": "解析一个C++项目的所有源文件(.cpp/.hpp/.c/.h)| 输入参数为路径",
+ "Function": HotReload(解析一个C项目)
+ },
+ "解析整个Go项目": {
+ "Group": "编程",
+ "Color": "stop",
+ "AsButton": False, # 加入下拉菜单中
+ "Info": "解析一个Go项目的所有源文件 | 输入参数为路径",
+ "Function": HotReload(解析一个Golang项目)
+ },
+ "解析整个Rust项目": {
+ "Group": "编程",
+ "Color": "stop",
+ "AsButton": False, # 加入下拉菜单中
+ "Info": "解析一个Rust项目的所有源文件 | 输入参数为路径",
+ "Function": HotReload(解析一个Rust项目)
+ },
+ "解析整个Java项目": {
+ "Group": "编程",
+ "Color": "stop",
+ "AsButton": False, # 加入下拉菜单中
+ "Info": "解析一个Java项目的所有源文件 | 输入参数为路径",
+ "Function": HotReload(解析一个Java项目)
+ },
+ "解析整个前端项目(js,ts,css等)": {
+ "Group": "编程",
+ "Color": "stop",
+ "AsButton": False, # 加入下拉菜单中
+ "Info": "解析一个前端项目的所有源文件(js,ts,css等) | 输入参数为路径",
+ "Function": HotReload(解析一个前端项目)
+ },
+ "解析整个Lua项目": {
+ "Group": "编程",
+ "Color": "stop",
+ "AsButton": False, # 加入下拉菜单中
+ "Info": "解析一个Lua项目的所有源文件 | 输入参数为路径",
+ "Function": HotReload(解析一个Lua项目)
+ },
+ "解析整个CSharp项目": {
+ "Group": "编程",
+ "Color": "stop",
+ "AsButton": False, # 加入下拉菜单中
+ "Info": "解析一个CSharp项目的所有源文件 | 输入参数为路径",
+ "Function": HotReload(解析一个CSharp项目)
+ },
+ "解析Jupyter Notebook文件": {
+ "Group": "编程",
+ "Color": "stop",
+ "AsButton": False,
+ "Info": "解析Jupyter Notebook文件 | 输入参数为路径",
+ "Function": HotReload(解析ipynb文件),
+ "AdvancedArgs": True, # 调用时,唤起高级参数输入区(默认False)
+ "ArgsReminder": "若输入0,则不解析notebook中的Markdown块", # 高级参数输入区的显示提示
+ },
+ "读Tex论文写摘要": {
+ "Group": "学术",
+ "Color": "stop",
+ "AsButton": False,
+ "Info": "读取Tex论文并写摘要 | 输入参数为路径",
+ "Function": HotReload(读文章写摘要)
+ },
+ "翻译README或MD": {
+ "Group": "编程",
+ "Color": "stop",
+ "AsButton": True,
+ "Info": "将Markdown翻译为中文 | 输入参数为路径或URL",
+ "Function": HotReload(Markdown英译中)
+ },
+ "翻译Markdown或README(支持Github链接)": {
+ "Group": "编程",
+ "Color": "stop",
+ "AsButton": False,
+ "Info": "将Markdown或README翻译为中文 | 输入参数为路径或URL",
+ "Function": HotReload(Markdown英译中)
+ },
+ "批量生成函数注释": {
+ "Group": "编程",
+ "Color": "stop",
+ "AsButton": False, # 加入下拉菜单中
+ "Info": "批量生成函数的注释 | 输入参数为路径",
+ "Function": HotReload(批量生成函数注释)
+ },
+ "保存当前的对话": {
+ "Group": "对话",
+ "AsButton": True,
+ "Info": "保存当前的对话 | 不需要输入参数",
+ "Function": HotReload(对话历史存档)
+ },
+ "[多线程Demo]解析此项目本身(源码自译解)": {
+ "Group": "对话|编程",
+ "AsButton": False, # 加入下拉菜单中
+ "Info": "多线程解析并翻译此项目的源码 | 不需要输入参数",
+ "Function": HotReload(解析项目本身)
+ },
+ "[插件demo]历史上的今天": {
+ "Group": "对话",
+ "AsButton": True,
+ "Info": "查看历史上的今天事件 | 不需要输入参数",
+ "Function": HotReload(高阶功能模板函数)
+ },
+ "精准翻译PDF论文": {
+ "Group": "学术",
+ "Color": "stop",
+ "AsButton": True,
+ "Info": "精准翻译PDF论文为中文 | 输入参数为路径",
"Function": HotReload(批量翻译PDF文档)
},
"询问多个GPT模型": {
- "Color": "stop", # 按钮颜色
+ "Group": "对话",
+ "Color": "stop",
+ "AsButton": True,
"Function": HotReload(同时问询)
},
- "[测试功能] 批量总结PDF文档": {
+ "批量总结PDF文档": {
+ "Group": "学术",
"Color": "stop",
"AsButton": False, # 加入下拉菜单中
- # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效
+ "Info": "批量总结PDF文档的内容 | 输入参数为路径",
"Function": HotReload(批量总结PDF文档)
},
- # "[测试功能] 批量总结PDF文档pdfminer": {
- # "Color": "stop",
- # "AsButton": False, # 加入下拉菜单中
- # "Function": HotReload(批量总结PDF文档pdfminer)
- # },
"谷歌学术检索助手(输入谷歌学术搜索页url)": {
+ "Group": "学术",
"Color": "stop",
"AsButton": False, # 加入下拉菜单中
+ "Info": "使用谷歌学术检索助手搜索指定URL的结果 | 输入参数为谷歌学术搜索页的URL",
"Function": HotReload(谷歌检索小助手)
},
"理解PDF文档内容 (模仿ChatPDF)": {
- # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效
+ "Group": "学术",
"Color": "stop",
"AsButton": False, # 加入下拉菜单中
+ "Info": "理解PDF文档的内容并进行回答 | 输入参数为路径",
"Function": HotReload(理解PDF文档内容标准文件输入)
},
"英文Latex项目全文润色(输入路径或上传压缩包)": {
- # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效
+ "Group": "学术",
"Color": "stop",
"AsButton": False, # 加入下拉菜单中
+ "Info": "对英文Latex项目全文进行润色处理 | 输入参数为路径或上传压缩包",
"Function": HotReload(Latex英文润色)
},
"英文Latex项目全文纠错(输入路径或上传压缩包)": {
- # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效
+ "Group": "学术",
"Color": "stop",
"AsButton": False, # 加入下拉菜单中
+ "Info": "对英文Latex项目全文进行纠错处理 | 输入参数为路径或上传压缩包",
"Function": HotReload(Latex英文纠错)
},
"中文Latex项目全文润色(输入路径或上传压缩包)": {
- # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效
+ "Group": "学术",
"Color": "stop",
"AsButton": False, # 加入下拉菜单中
+ "Info": "对中文Latex项目全文进行润色处理 | 输入参数为路径或上传压缩包",
"Function": HotReload(Latex中文润色)
},
"Latex项目全文中译英(输入路径或上传压缩包)": {
- # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效
+ "Group": "学术",
"Color": "stop",
"AsButton": False, # 加入下拉菜单中
+ "Info": "对Latex项目全文进行中译英处理 | 输入参数为路径或上传压缩包",
"Function": HotReload(Latex中译英)
},
"Latex项目全文英译中(输入路径或上传压缩包)": {
- # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效
+ "Group": "学术",
"Color": "stop",
"AsButton": False, # 加入下拉菜单中
+ "Info": "对Latex项目全文进行英译中处理 | 输入参数为路径或上传压缩包",
"Function": HotReload(Latex英译中)
},
"批量Markdown中译英(输入路径或上传压缩包)": {
- # HotReload 的意思是热更新,修改函数插件代码后,不需要重启程序,代码直接生效
+ "Group": "编程",
"Color": "stop",
"AsButton": False, # 加入下拉菜单中
+ "Info": "批量将Markdown文件中文翻译为英文 | 输入参数为路径或上传压缩包",
"Function": HotReload(Markdown中译英)
},
+ }
-
- })
-
- ###################### 第三组插件 ###########################
- # [第三组插件]: 尚未充分测试的函数插件
-
+ # -=--=- 尚未充分测试的实验性插件 & 需要额外依赖的插件 -=--=-
try:
from crazy_functions.下载arxiv论文翻译摘要 import 下载arxiv论文并翻译摘要
function_plugins.update({
"一键下载arxiv论文并翻译摘要(先在input输入编号,如1812.10695)": {
+ "Group": "学术",
"Color": "stop",
"AsButton": False, # 加入下拉菜单中
+ # "Info": "下载arxiv论文并翻译摘要 | 输入参数为arxiv编号如1812.10695",
"Function": HotReload(下载arxiv论文并翻译摘要)
}
})
@@ -233,16 +286,20 @@ def get_crazy_functions():
from crazy_functions.联网的ChatGPT import 连接网络回答问题
function_plugins.update({
"连接网络回答问题(输入问题后点击该插件,需要访问谷歌)": {
+ "Group": "对话",
"Color": "stop",
"AsButton": False, # 加入下拉菜单中
+ # "Info": "连接网络回答问题(需要访问谷歌)| 输入参数是一个问题",
"Function": HotReload(连接网络回答问题)
}
})
from crazy_functions.联网的ChatGPT_bing版 import 连接bing搜索回答问题
function_plugins.update({
"连接网络回答问题(中文Bing版,输入问题后点击该插件)": {
+ "Group": "对话",
"Color": "stop",
"AsButton": False, # 加入下拉菜单中
+ "Info": "连接网络回答问题(需要访问中文Bing)| 输入参数是一个问题",
"Function": HotReload(连接bing搜索回答问题)
}
})
@@ -253,10 +310,11 @@ def get_crazy_functions():
from crazy_functions.解析项目源代码 import 解析任意code项目
function_plugins.update({
"解析项目源代码(手动指定和筛选源代码文件类型)": {
+ "Group": "编程",
"Color": "stop",
"AsButton": False,
- "AdvancedArgs": True, # 调用时,唤起高级参数输入区(默认False)
- "ArgsReminder": "输入时用逗号隔开, *代表通配符, 加了^代表不匹配; 不输入代表全部匹配。例如: \"*.c, ^*.cpp, config.toml, ^*.toml\"", # 高级参数输入区的显示提示
+ "AdvancedArgs": True, # 调用时,唤起高级参数输入区(默认False)
+ "ArgsReminder": "输入时用逗号隔开, *代表通配符, 加了^代表不匹配; 不输入代表全部匹配。例如: \"*.c, ^*.cpp, config.toml, ^*.toml\"", # 高级参数输入区的显示提示
"Function": HotReload(解析任意code项目)
},
})
@@ -267,10 +325,11 @@ def get_crazy_functions():
from crazy_functions.询问多个大语言模型 import 同时问询_指定模型
function_plugins.update({
"询问多个GPT模型(手动指定询问哪些模型)": {
+ "Group": "对话",
"Color": "stop",
"AsButton": False,
- "AdvancedArgs": True, # 调用时,唤起高级参数输入区(默认False)
- "ArgsReminder": "支持任意数量的llm接口,用&符号分隔。例如chatglm&gpt-3.5-turbo&api2d-gpt-4", # 高级参数输入区的显示提示
+ "AdvancedArgs": True, # 调用时,唤起高级参数输入区(默认False)
+ "ArgsReminder": "支持任意数量的llm接口,用&符号分隔。例如chatglm&gpt-3.5-turbo&api2d-gpt-4", # 高级参数输入区的显示提示
"Function": HotReload(同时问询_指定模型)
},
})
@@ -281,10 +340,12 @@ def get_crazy_functions():
from crazy_functions.图片生成 import 图片生成
function_plugins.update({
"图片生成(先切换模型到openai或api2d)": {
+ "Group": "对话",
"Color": "stop",
"AsButton": False,
- "AdvancedArgs": True, # 调用时,唤起高级参数输入区(默认False)
- "ArgsReminder": "在这里输入分辨率, 如256x256(默认)", # 高级参数输入区的显示提示
+ "AdvancedArgs": True, # 调用时,唤起高级参数输入区(默认False)
+ "ArgsReminder": "在这里输入分辨率, 如256x256(默认)", # 高级参数输入区的显示提示
+ "Info": "图片生成 | 输入参数字符串,提供图像的内容",
"Function": HotReload(图片生成)
},
})
@@ -295,10 +356,12 @@ def get_crazy_functions():
from crazy_functions.总结音视频 import 总结音视频
function_plugins.update({
"批量总结音视频(输入路径或上传压缩包)": {
+ "Group": "对话",
"Color": "stop",
"AsButton": False,
"AdvancedArgs": True,
"ArgsReminder": "调用openai api 使用whisper-1模型, 目前支持的格式:mp4, m4a, wav, mpga, mpeg, mp3。此处可以输入解析提示,例如:解析为简体中文(默认)。",
+ "Info": "批量总结音频或视频 | 输入参数为路径",
"Function": HotReload(总结音视频)
}
})
@@ -309,8 +372,10 @@ def get_crazy_functions():
from crazy_functions.数学动画生成manim import 动画生成
function_plugins.update({
"数学动画生成(Manim)": {
+ "Group": "对话",
"Color": "stop",
"AsButton": False,
+ "Info": "按照自然语言描述生成一个动画 | 输入参数是一段话",
"Function": HotReload(动画生成)
}
})
@@ -321,6 +386,7 @@ def get_crazy_functions():
from crazy_functions.批量Markdown翻译 import Markdown翻译指定语言
function_plugins.update({
"Markdown翻译(手动指定语言)": {
+ "Group": "编程",
"Color": "stop",
"AsButton": False,
"AdvancedArgs": True,
@@ -335,6 +401,7 @@ def get_crazy_functions():
from crazy_functions.Langchain知识库 import 知识库问答
function_plugins.update({
"构建知识库(请先上传文件素材)": {
+ "Group": "对话",
"Color": "stop",
"AsButton": False,
"AdvancedArgs": True,
@@ -349,6 +416,7 @@ def get_crazy_functions():
from crazy_functions.Langchain知识库 import 读取知识库作答
function_plugins.update({
"知识库问答": {
+ "Group": "对话",
"Color": "stop",
"AsButton": False,
"AdvancedArgs": True,
@@ -358,11 +426,12 @@ def get_crazy_functions():
})
except:
print('Load function plugin failed')
-
+
try:
from crazy_functions.交互功能函数模板 import 交互功能模板函数
function_plugins.update({
"交互功能模板函数": {
+ "Group": "对话",
"Color": "stop",
"AsButton": False,
"Function": HotReload(交互功能模板函数)
@@ -371,6 +440,94 @@ def get_crazy_functions():
except:
print('Load function plugin failed')
+ try:
+ from crazy_functions.Latex输出PDF结果 import Latex英文纠错加PDF对比
+ function_plugins.update({
+ "Latex英文纠错+高亮修正位置 [需Latex]": {
+ "Group": "学术",
+ "Color": "stop",
+ "AsButton": False,
+ "AdvancedArgs": True,
+ "ArgsReminder": "如果有必要, 请在此处追加更细致的矫错指令(使用英文)。",
+ "Function": HotReload(Latex英文纠错加PDF对比)
+ }
+ })
+ from crazy_functions.Latex输出PDF结果 import Latex翻译中文并重新编译PDF
+ function_plugins.update({
+ "Arixv论文精细翻译(输入arxivID)[需Latex]": {
+ "Group": "学术",
+ "Color": "stop",
+ "AsButton": False,
+ "AdvancedArgs": True,
+ "ArgsReminder":
+ "如果有必要, 请在此处给出自定义翻译命令, 解决部分词汇翻译不准确的问题。 " +
+ "例如当单词'agent'翻译不准确时, 请尝试把以下指令复制到高级参数区: " +
+ 'If the term "agent" is used in this section, it should be translated to "智能体". ',
+ "Info": "Arixv论文精细翻译 | 输入参数arxiv论文的ID,比如1812.10695",
+ "Function": HotReload(Latex翻译中文并重新编译PDF)
+ }
+ })
+ function_plugins.update({
+ "本地Latex论文精细翻译(上传Latex项目)[需Latex]": {
+ "Group": "学术",
+ "Color": "stop",
+ "AsButton": False,
+ "AdvancedArgs": True,
+ "ArgsReminder":
+ "如果有必要, 请在此处给出自定义翻译命令, 解决部分词汇翻译不准确的问题。 " +
+ "例如当单词'agent'翻译不准确时, 请尝试把以下指令复制到高级参数区: " +
+ 'If the term "agent" is used in this section, it should be translated to "智能体". ',
+ "Info": "本地Latex论文精细翻译 | 输入参数是路径",
+ "Function": HotReload(Latex翻译中文并重新编译PDF)
+ }
+ })
+ except:
+ print('Load function plugin failed')
+
+ try:
+ from toolbox import get_conf
+ ENABLE_AUDIO, = get_conf('ENABLE_AUDIO')
+ if ENABLE_AUDIO:
+ from crazy_functions.语音助手 import 语音助手
+ function_plugins.update({
+ "实时音频采集": {
+ "Group": "对话",
+ "Color": "stop",
+ "AsButton": True,
+ "Info": "开始语言对话 | 没有输入参数",
+ "Function": HotReload(语音助手)
+ }
+ })
+ except:
+ print('Load function plugin failed')
+
+ try:
+ from crazy_functions.批量翻译PDF文档_NOUGAT import 批量翻译PDF文档
+ function_plugins.update({
+ "精准翻译PDF文档(NOUGAT)": {
+ "Group": "学术",
+ "Color": "stop",
+ "AsButton": False,
+ "Function": HotReload(批量翻译PDF文档)
+ }
+ })
+ except:
+ print('Load function plugin failed')
+
+
+ # try:
+ # from crazy_functions.CodeInterpreter import 虚空终端CodeInterpreter
+ # function_plugins.update({
+ # "CodeInterpreter(开发中,仅供测试)": {
+ # "Group": "编程|对话",
+ # "Color": "stop",
+ # "AsButton": False,
+ # "Function": HotReload(虚空终端CodeInterpreter)
+ # }
+ # })
+ # except:
+ # print('Load function plugin failed')
+
# try:
# from crazy_functions.chatglm微调工具 import 微调数据集生成
# function_plugins.update({
@@ -385,55 +542,23 @@ def get_crazy_functions():
# except:
# print('Load function plugin failed')
- try:
- from crazy_functions.Latex输出PDF结果 import Latex英文纠错加PDF对比
- function_plugins.update({
- "Latex英文纠错+高亮修正位置 [需Latex]": {
- "Color": "stop",
- "AsButton": False,
- "AdvancedArgs": True,
- "ArgsReminder": "如果有必要, 请在此处追加更细致的矫错指令(使用英文)。",
- "Function": HotReload(Latex英文纠错加PDF对比)
- }
- })
- from crazy_functions.Latex输出PDF结果 import Latex翻译中文并重新编译PDF
- function_plugins.update({
- "Arixv论文精细翻译(输入arxivID)[需Latex]": {
- "Color": "stop",
- "AsButton": False,
- "AdvancedArgs": True,
- "ArgsReminder":
- "如果有必要, 请在此处给出自定义翻译命令, 解决部分词汇翻译不准确的问题。 "+
- "例如当单词'agent'翻译不准确时, 请尝试把以下指令复制到高级参数区: " + 'If the term "agent" is used in this section, it should be translated to "智能体". ',
- "Function": HotReload(Latex翻译中文并重新编译PDF)
- }
- })
- function_plugins.update({
- "本地Latex论文精细翻译(上传Latex项目)[需Latex]": {
- "Color": "stop",
- "AsButton": False,
- "AdvancedArgs": True,
- "ArgsReminder":
- "如果有必要, 请在此处给出自定义翻译命令, 解决部分词汇翻译不准确的问题。 "+
- "例如当单词'agent'翻译不准确时, 请尝试把以下指令复制到高级参数区: " + 'If the term "agent" is used in this section, it should be translated to "智能体". ',
- "Function": HotReload(Latex翻译中文并重新编译PDF)
- }
- })
- except:
- print('Load function plugin failed')
- # try:
- # from crazy_functions.虚空终端 import 终端
- # function_plugins.update({
- # "超级终端": {
- # "Color": "stop",
- # "AsButton": False,
- # # "AdvancedArgs": True,
- # # "ArgsReminder": "",
- # "Function": HotReload(终端)
- # }
- # })
- # except:
- # print('Load function plugin failed')
+
+ """
+ 设置默认值:
+ - 默认 Group = 对话
+ - 默认 AsButton = True
+ - 默认 AdvancedArgs = False
+ - 默认 Color = secondary
+ """
+ for name, function_meta in function_plugins.items():
+ if "Group" not in function_meta:
+ function_plugins[name]["Group"] = '对话'
+ if "AsButton" not in function_meta:
+ function_plugins[name]["AsButton"] = True
+ if "AdvancedArgs" not in function_meta:
+ function_plugins[name]["AdvancedArgs"] = False
+ if "Color" not in function_meta:
+ function_plugins[name]["Color"] = 'secondary'
return function_plugins
diff --git a/crazy_functions/CodeInterpreter.py b/crazy_functions/CodeInterpreter.py
new file mode 100644
index 00000000..3c970f35
--- /dev/null
+++ b/crazy_functions/CodeInterpreter.py
@@ -0,0 +1,231 @@
+from collections.abc import Callable, Iterable, Mapping
+from typing import Any
+from toolbox import CatchException, update_ui, gen_time_str, trimmed_format_exc, promote_file_to_downloadzone, clear_file_downloadzone
+from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
+from .crazy_utils import input_clipping, try_install_deps
+from multiprocessing import Process, Pipe
+import os
+import time
+
+templete = """
+```python
+import ... # Put dependencies here, e.g. import numpy as np
+
+class TerminalFunction(object): # Do not change the name of the class, The name of the class must be `TerminalFunction`
+
+ def run(self, path): # The name of the function must be `run`, it takes only a positional argument.
+ # rewrite the function you have just written here
+ ...
+ return generated_file_path
+```
+"""
+
+def inspect_dependency(chatbot, history):
+ yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
+ return True
+
+def get_code_block(reply):
+ import re
+ pattern = r"```([\s\S]*?)```" # regex pattern to match code blocks
+ matches = re.findall(pattern, reply) # find all code blocks in text
+ if len(matches) == 1:
+ return matches[0].strip('python') # code block
+ for match in matches:
+ if 'class TerminalFunction' in match:
+ return match.strip('python') # code block
+ raise RuntimeError("GPT is not generating proper code.")
+
+def gpt_interact_multi_step(txt, file_type, llm_kwargs, chatbot, history):
+ # 输入
+ prompt_compose = [
+ f'Your job:\n'
+ f'1. write a single Python function, which takes a path of a `{file_type}` file as the only argument and returns a `string` containing the result of analysis or the path of generated files. \n',
+ f"2. You should write this function to perform following task: " + txt + "\n",
+ f"3. Wrap the output python function with markdown codeblock."
+ ]
+ i_say = "".join(prompt_compose)
+ demo = []
+
+ # 第一步
+ gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(
+ inputs=i_say, inputs_show_user=i_say,
+ llm_kwargs=llm_kwargs, chatbot=chatbot, history=demo,
+ sys_prompt= r"You are a programmer."
+ )
+ history.extend([i_say, gpt_say])
+ yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 界面更新
+
+ # 第二步
+ prompt_compose = [
+ "If previous stage is successful, rewrite the function you have just written to satisfy following templete: \n",
+ templete
+ ]
+ i_say = "".join(prompt_compose); inputs_show_user = "If previous stage is successful, rewrite the function you have just written to satisfy executable templete. "
+ gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(
+ inputs=i_say, inputs_show_user=inputs_show_user,
+ llm_kwargs=llm_kwargs, chatbot=chatbot, history=history,
+ sys_prompt= r"You are a programmer."
+ )
+ code_to_return = gpt_say
+ history.extend([i_say, gpt_say])
+ yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 界面更新
+
+ # # 第三步
+ # i_say = "Please list to packages to install to run the code above. Then show me how to use `try_install_deps` function to install them."
+ # i_say += 'For instance. `try_install_deps(["opencv-python", "scipy", "numpy"])`'
+ # installation_advance = yield from request_gpt_model_in_new_thread_with_ui_alive(
+ # inputs=i_say, inputs_show_user=inputs_show_user,
+ # llm_kwargs=llm_kwargs, chatbot=chatbot, history=history,
+ # sys_prompt= r"You are a programmer."
+ # )
+ # # # 第三步
+ # i_say = "Show me how to use `pip` to install packages to run the code above. "
+ # i_say += 'For instance. `pip install -r opencv-python scipy numpy`'
+ # installation_advance = yield from request_gpt_model_in_new_thread_with_ui_alive(
+ # inputs=i_say, inputs_show_user=i_say,
+ # llm_kwargs=llm_kwargs, chatbot=chatbot, history=history,
+ # sys_prompt= r"You are a programmer."
+ # )
+ installation_advance = ""
+
+ return code_to_return, installation_advance, txt, file_type, llm_kwargs, chatbot, history
+
+def make_module(code):
+ module_file = 'gpt_fn_' + gen_time_str().replace('-','_')
+ with open(f'gpt_log/{module_file}.py', 'w', encoding='utf8') as f:
+ f.write(code)
+
+ def get_class_name(class_string):
+ import re
+ # Use regex to extract the class name
+ class_name = re.search(r'class (\w+)\(', class_string).group(1)
+ return class_name
+
+ class_name = get_class_name(code)
+ return f"gpt_log.{module_file}->{class_name}"
+
+def init_module_instance(module):
+ import importlib
+ module_, class_ = module.split('->')
+ init_f = getattr(importlib.import_module(module_), class_)
+ return init_f()
+
+def for_immediate_show_off_when_possible(file_type, fp, chatbot):
+ if file_type in ['png', 'jpg']:
+ image_path = os.path.abspath(fp)
+ chatbot.append(['这是一张图片, 展示如下:',
+ f'本地文件地址:
`{image_path}`
'+
+ f'本地文件预览:

'
+ ])
+ return chatbot
+
+def subprocess_worker(instance, file_path, return_dict):
+ return_dict['result'] = instance.run(file_path)
+
+def have_any_recent_upload_files(chatbot):
+ _5min = 5 * 60
+ if not chatbot: return False # chatbot is None
+ most_recent_uploaded = chatbot._cookies.get("most_recent_uploaded", None)
+ if not most_recent_uploaded: return False # most_recent_uploaded is None
+ if time.time() - most_recent_uploaded["time"] < _5min: return True # most_recent_uploaded is new
+ else: return False # most_recent_uploaded is too old
+
+def get_recent_file_prompt_support(chatbot):
+ most_recent_uploaded = chatbot._cookies.get("most_recent_uploaded", None)
+ path = most_recent_uploaded['path']
+ return path
+
+@CatchException
+def 虚空终端CodeInterpreter(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port):
+ """
+ txt 输入栏用户输入的文本,例如需要翻译的一段话,再例如一个包含了待处理文件的路径
+ llm_kwargs gpt模型参数,如温度和top_p等,一般原样传递下去就行
+ plugin_kwargs 插件模型的参数,暂时没有用武之地
+ chatbot 聊天显示框的句柄,用于显示给用户
+ history 聊天历史,前情提要
+ system_prompt 给gpt的静默提醒
+ web_port 当前软件运行的端口号
+ """
+ raise NotImplementedError
+
+ # 清空历史,以免输入溢出
+ history = []; clear_file_downloadzone(chatbot)
+
+ # 基本信息:功能、贡献者
+ chatbot.append([
+ "函数插件功能?",
+ "CodeInterpreter开源版, 此插件处于开发阶段, 建议暂时不要使用, 插件初始化中 ..."
+ ])
+ yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
+
+ if have_any_recent_upload_files(chatbot):
+ file_path = get_recent_file_prompt_support(chatbot)
+ else:
+ chatbot.append(["文件检索", "没有发现任何近期上传的文件。"])
+ yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
+
+ # 读取文件
+ if ("recently_uploaded_files" in plugin_kwargs) and (plugin_kwargs["recently_uploaded_files"] == ""): plugin_kwargs.pop("recently_uploaded_files")
+ recently_uploaded_files = plugin_kwargs.get("recently_uploaded_files", None)
+ file_path = recently_uploaded_files[-1]
+ file_type = file_path.split('.')[-1]
+
+ # 粗心检查
+ if 'private_upload' in txt:
+ chatbot.append([
+ "...",
+ f"请在输入框内填写需求,然后再次点击该插件(文件路径 {file_path} 已经被记忆)"
+ ])
+ yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
+ return
+
+ # 开始干正事
+ for j in range(5): # 最多重试5次
+ try:
+ code, installation_advance, txt, file_type, llm_kwargs, chatbot, history = \
+ yield from gpt_interact_multi_step(txt, file_type, llm_kwargs, chatbot, history)
+ code = get_code_block(code)
+ res = make_module(code)
+ instance = init_module_instance(res)
+ break
+ except Exception as e:
+ chatbot.append([f"第{j}次代码生成尝试,失败了", f"错误追踪\n```\n{trimmed_format_exc()}\n```\n"])
+ yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
+
+ # 代码生成结束, 开始执行
+ try:
+ import multiprocessing
+ manager = multiprocessing.Manager()
+ return_dict = manager.dict()
+
+ p = multiprocessing.Process(target=subprocess_worker, args=(instance, file_path, return_dict))
+ # only has 10 seconds to run
+ p.start(); p.join(timeout=10)
+ if p.is_alive(): p.terminate(); p.join()
+ p.close()
+ res = return_dict['result']
+ # res = instance.run(file_path)
+ except Exception as e:
+ chatbot.append(["执行失败了", f"错误追踪\n```\n{trimmed_format_exc()}\n```\n"])
+ # chatbot.append(["如果是缺乏依赖,请参考以下建议", installation_advance])
+ yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
+ return
+
+ # 顺利完成,收尾
+ res = str(res)
+ if os.path.exists(res):
+ chatbot.append(["执行成功了,结果是一个有效文件", "结果:" + res])
+ new_file_path = promote_file_to_downloadzone(res, chatbot=chatbot)
+ chatbot = for_immediate_show_off_when_possible(file_type, new_file_path, chatbot)
+ yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 界面更新
+ else:
+ chatbot.append(["执行成功了,结果是一个字符串", "结果:" + res])
+ yield from update_ui(chatbot=chatbot, history=history) # 刷新界面 # 界面更新
+
+"""
+测试:
+ 裁剪图像,保留下半部分
+ 交换图像的蓝色通道和红色通道
+ 将图像转为灰度图像
+ 将csv文件转excel表格
+"""
\ No newline at end of file
diff --git a/crazy_functions/Latex输出PDF结果.py b/crazy_functions/Latex输出PDF结果.py
index e79cf822..8686f7e9 100644
--- a/crazy_functions/Latex输出PDF结果.py
+++ b/crazy_functions/Latex输出PDF结果.py
@@ -6,7 +6,7 @@ pj = os.path.join
ARXIV_CACHE_DIR = os.path.expanduser(f"~/arxiv_cache/")
# =================================== 工具函数 ===============================================
-专业词汇声明 = 'If the term "agent" is used in this section, it should be translated to "智能体". '
+# 专业词汇声明 = 'If the term "agent" is used in this section, it should be translated to "智能体". '
def switch_prompt(pfg, mode, more_requirement):
"""
Generate prompts and system prompts based on the mode for proofreading or translating.
@@ -109,7 +109,7 @@ def arxiv_download(chatbot, history, txt):
url_ = txt # https://arxiv.org/abs/1707.06690
if not txt.startswith('https://arxiv.org/abs/'):
- msg = f"解析arxiv网址失败, 期望格式例如: https://arxiv.org/abs/1707.06690。实际得到格式: {url_}"
+ msg = f"解析arxiv网址失败, 期望格式例如: https://arxiv.org/abs/1707.06690。实际得到格式: {url_}。"
yield from update_ui_lastest_msg(msg, chatbot=chatbot, history=history) # 刷新界面
return msg, None
# <-------------- set format ------------->
@@ -255,7 +255,7 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot,
project_folder = txt
else:
if txt == "": txt = '空空如也的输入栏'
- report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}")
+ report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无法处理: {txt}")
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
return
@@ -291,7 +291,7 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot,
yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面
promote_file_to_downloadzone(file=zip_res, chatbot=chatbot)
else:
- chatbot.append((f"失败了", '虽然PDF生成失败了, 但请查收结果(压缩包), 内含已经翻译的Tex文档, 也是可读的, 您可以到Github Issue区, 用该压缩包+对话历史存档进行反馈 ...'))
+ chatbot.append((f"失败了", '虽然PDF生成失败了, 但请查收结果(压缩包), 内含已经翻译的Tex文档, 您可以到Github Issue区, 用该压缩包进行反馈。如系统是Linux,请检查系统字体(见Github wiki) ...'))
yield from update_ui(chatbot=chatbot, history=history); time.sleep(1) # 刷新界面
promote_file_to_downloadzone(file=zip_res, chatbot=chatbot)
diff --git a/crazy_functions/crazy_utils.py b/crazy_functions/crazy_utils.py
index ffe95e2b..5a314b37 100644
--- a/crazy_functions/crazy_utils.py
+++ b/crazy_functions/crazy_utils.py
@@ -591,11 +591,16 @@ def get_files_from_everything(txt, type): # type='.md'
# 网络的远程文件
import requests
from toolbox import get_conf
+ from toolbox import get_log_folder, gen_time_str
proxies, = get_conf('proxies')
- r = requests.get(txt, proxies=proxies)
- with open('./gpt_log/temp'+type, 'wb+') as f: f.write(r.content)
- project_folder = './gpt_log/'
- file_manifest = ['./gpt_log/temp'+type]
+ try:
+ r = requests.get(txt, proxies=proxies)
+ except:
+ raise ConnectionRefusedError(f"无法下载资源{txt},请检查。")
+ path = os.path.join(get_log_folder(plugin_name='web_download'), gen_time_str()+type)
+ with open(path, 'wb+') as f: f.write(r.content)
+ project_folder = get_log_folder(plugin_name='web_download')
+ file_manifest = [path]
elif txt.endswith(type):
# 直接给定文件
file_manifest = [txt]
diff --git a/crazy_functions/json_fns/pydantic_io.py b/crazy_functions/json_fns/pydantic_io.py
new file mode 100644
index 00000000..4e300d65
--- /dev/null
+++ b/crazy_functions/json_fns/pydantic_io.py
@@ -0,0 +1,111 @@
+"""
+https://github.com/langchain-ai/langchain/blob/master/docs/extras/modules/model_io/output_parsers/pydantic.ipynb
+
+Example 1.
+
+# Define your desired data structure.
+class Joke(BaseModel):
+ setup: str = Field(description="question to set up a joke")
+ punchline: str = Field(description="answer to resolve the joke")
+
+ # You can add custom validation logic easily with Pydantic.
+ @validator("setup")
+ def question_ends_with_question_mark(cls, field):
+ if field[-1] != "?":
+ raise ValueError("Badly formed question!")
+ return field
+
+
+Example 2.
+
+# Here's another example, but with a compound typed field.
+class Actor(BaseModel):
+ name: str = Field(description="name of an actor")
+ film_names: List[str] = Field(description="list of names of films they starred in")
+"""
+
+import json, re, logging
+
+
+PYDANTIC_FORMAT_INSTRUCTIONS = """The output should be formatted as a JSON instance that conforms to the JSON schema below.
+
+As an example, for the schema {{"properties": {{"foo": {{"title": "Foo", "description": "a list of strings", "type": "array", "items": {{"type": "string"}}}}}}, "required": ["foo"]}}
+the object {{"foo": ["bar", "baz"]}} is a well-formatted instance of the schema. The object {{"properties": {{"foo": ["bar", "baz"]}}}} is not well-formatted.
+
+Here is the output schema:
+```
+{schema}
+```"""
+
+
+PYDANTIC_FORMAT_INSTRUCTIONS_SIMPLE = """The output should be formatted as a JSON instance that conforms to the JSON schema below.
+```
+{schema}
+```"""
+
+class JsonStringError(Exception): ...
+
+class GptJsonIO():
+
+ def __init__(self, schema, example_instruction=True):
+ self.pydantic_object = schema
+ self.example_instruction = example_instruction
+ self.format_instructions = self.generate_format_instructions()
+
+ def generate_format_instructions(self):
+ schema = self.pydantic_object.schema()
+
+ # Remove extraneous fields.
+ reduced_schema = schema
+ if "title" in reduced_schema:
+ del reduced_schema["title"]
+ if "type" in reduced_schema:
+ del reduced_schema["type"]
+ # Ensure json in context is well-formed with double quotes.
+ if self.example_instruction:
+ schema_str = json.dumps(reduced_schema)
+ return PYDANTIC_FORMAT_INSTRUCTIONS.format(schema=schema_str)
+ else:
+ return PYDANTIC_FORMAT_INSTRUCTIONS_SIMPLE.format(schema=schema_str)
+
+ def generate_output(self, text):
+ # Greedy search for 1st json candidate.
+ match = re.search(
+ r"\{.*\}", text.strip(), re.MULTILINE | re.IGNORECASE | re.DOTALL
+ )
+ json_str = ""
+ if match: json_str = match.group()
+ json_object = json.loads(json_str, strict=False)
+ final_object = self.pydantic_object.parse_obj(json_object)
+ return final_object
+
+ def generate_repair_prompt(self, broken_json, error):
+ prompt = "Fix a broken json string.\n\n" + \
+ "(1) The broken json string need to fix is: \n\n" + \
+ "```" + "\n" + \
+ broken_json + "\n" + \
+ "```" + "\n\n" + \
+ "(2) The error message is: \n\n" + \
+ error + "\n\n" + \
+ "Now, fix this json string. \n\n"
+ return prompt
+
+ def generate_output_auto_repair(self, response, gpt_gen_fn):
+ """
+ response: string containing canidate json
+ gpt_gen_fn: gpt_gen_fn(inputs, sys_prompt)
+ """
+ try:
+ result = self.generate_output(response)
+ except Exception as e:
+ try:
+ logging.info(f'Repairing json:{response}')
+ repair_prompt = self.generate_repair_prompt(broken_json = response, error=repr(e))
+ result = self.generate_output(gpt_gen_fn(repair_prompt, self.format_instructions))
+ logging.info('Repaire json success.')
+ except Exception as e:
+ # 没辙了,放弃治疗
+ logging.info('Repaire json fail.')
+ raise JsonStringError('Cannot repair json.', str(e))
+ return result
+
diff --git a/crazy_functions/live_audio/aliyunASR.py b/crazy_functions/live_audio/aliyunASR.py
index 96410577..ed67fcd3 100644
--- a/crazy_functions/live_audio/aliyunASR.py
+++ b/crazy_functions/live_audio/aliyunASR.py
@@ -1,4 +1,4 @@
-import time, threading, json
+import time, logging, json
class AliyunASR():
@@ -12,14 +12,14 @@ class AliyunASR():
message = json.loads(message)
self.parsed_sentence = message['payload']['result']
self.event_on_entence_end.set()
- print(self.parsed_sentence)
+ # print(self.parsed_sentence)
def test_on_start(self, message, *args):
# print("test_on_start:{}".format(message))
pass
def test_on_error(self, message, *args):
- print("on_error args=>{}".format(args))
+ logging.error("on_error args=>{}".format(args))
pass
def test_on_close(self, *args):
@@ -36,7 +36,6 @@ class AliyunASR():
# print("on_completed:args=>{} message=>{}".format(args, message))
pass
-
def audio_convertion_thread(self, uuid):
# 在一个异步线程中采集音频
import nls # pip install git+https://github.com/aliyun/alibabacloud-nls-python-sdk.git
diff --git a/crazy_functions/pdf_fns/parse_pdf.py b/crazy_functions/pdf_fns/parse_pdf.py
index 00016be5..8a7117ad 100644
--- a/crazy_functions/pdf_fns/parse_pdf.py
+++ b/crazy_functions/pdf_fns/parse_pdf.py
@@ -20,6 +20,11 @@ def get_avail_grobid_url():
def parse_pdf(pdf_path, grobid_url):
import scipdf # pip install scipdf_parser
if grobid_url.endswith('/'): grobid_url = grobid_url.rstrip('/')
- article_dict = scipdf.parse_pdf_to_dict(pdf_path, grobid_url=grobid_url)
+ try:
+ article_dict = scipdf.parse_pdf_to_dict(pdf_path, grobid_url=grobid_url)
+ except GROBID_OFFLINE_EXCEPTION:
+ raise GROBID_OFFLINE_EXCEPTION("GROBID服务不可用,请修改config中的GROBID_URL,可修改成本地GROBID服务。")
+ except:
+ raise RuntimeError("解析PDF失败,请检查PDF是否损坏。")
return article_dict
diff --git a/crazy_functions/test_project/cpp/cppipc/buffer.cpp b/crazy_functions/test_project/cpp/cppipc/buffer.cpp
deleted file mode 100644
index 084b8153..00000000
--- a/crazy_functions/test_project/cpp/cppipc/buffer.cpp
+++ /dev/null
@@ -1,87 +0,0 @@
-#include "libipc/buffer.h"
-#include "libipc/utility/pimpl.h"
-
-#include
-
-namespace ipc {
-
-bool operator==(buffer const & b1, buffer const & b2) {
- return (b1.size() == b2.size()) && (std::memcmp(b1.data(), b2.data(), b1.size()) == 0);
-}
-
-bool operator!=(buffer const & b1, buffer const & b2) {
- return !(b1 == b2);
-}
-
-class buffer::buffer_ : public pimpl {
-public:
- void* p_;
- std::size_t s_;
- void* a_;
- buffer::destructor_t d_;
-
- buffer_(void* p, std::size_t s, buffer::destructor_t d, void* a)
- : p_(p), s_(s), a_(a), d_(d) {
- }
-
- ~buffer_() {
- if (d_ == nullptr) return;
- d_((a_ == nullptr) ? p_ : a_, s_);
- }
-};
-
-buffer::buffer()
- : buffer(nullptr, 0, nullptr, nullptr) {
-}
-
-buffer::buffer(void* p, std::size_t s, destructor_t d)
- : p_(p_->make(p, s, d, nullptr)) {
-}
-
-buffer::buffer(void* p, std::size_t s, destructor_t d, void* additional)
- : p_(p_->make(p, s, d, additional)) {
-}
-
-buffer::buffer(void* p, std::size_t s)
- : buffer(p, s, nullptr) {
-}
-
-buffer::buffer(char const & c)
- : buffer(const_cast(&c), 1) {
-}
-
-buffer::buffer(buffer&& rhs)
- : buffer() {
- swap(rhs);
-}
-
-buffer::~buffer() {
- p_->clear();
-}
-
-void buffer::swap(buffer& rhs) {
- std::swap(p_, rhs.p_);
-}
-
-buffer& buffer::operator=(buffer rhs) {
- swap(rhs);
- return *this;
-}
-
-bool buffer::empty() const noexcept {
- return (impl(p_)->p_ == nullptr) || (impl(p_)->s_ == 0);
-}
-
-void* buffer::data() noexcept {
- return impl(p_)->p_;
-}
-
-void const * buffer::data() const noexcept {
- return impl(p_)->p_;
-}
-
-std::size_t buffer::size() const noexcept {
- return impl(p_)->s_;
-}
-
-} // namespace ipc
diff --git a/crazy_functions/test_project/cpp/cppipc/ipc.cpp b/crazy_functions/test_project/cpp/cppipc/ipc.cpp
deleted file mode 100644
index 4dc71c07..00000000
--- a/crazy_functions/test_project/cpp/cppipc/ipc.cpp
+++ /dev/null
@@ -1,701 +0,0 @@
-
-#include
-#include
-#include
-#include // std::pair, std::move, std::forward
-#include
-#include // aligned_storage_t
-#include
-#include
-#include
-#include
-
-#include "libipc/ipc.h"
-#include "libipc/def.h"
-#include "libipc/shm.h"
-#include "libipc/pool_alloc.h"
-#include "libipc/queue.h"
-#include "libipc/policy.h"
-#include "libipc/rw_lock.h"
-#include "libipc/waiter.h"
-
-#include "libipc/utility/log.h"
-#include "libipc/utility/id_pool.h"
-#include "libipc/utility/scope_guard.h"
-#include "libipc/utility/utility.h"
-
-#include "libipc/memory/resource.h"
-#include "libipc/platform/detail.h"
-#include "libipc/circ/elem_array.h"
-
-namespace {
-
-using msg_id_t = std::uint32_t;
-using acc_t = std::atomic;
-
-template
-struct msg_t;
-
-template
-struct msg_t<0, AlignSize> {
- msg_id_t cc_id_;
- msg_id_t id_;
- std::int32_t remain_;
- bool storage_;
-};
-
-template
-struct msg_t : msg_t<0, AlignSize> {
- std::aligned_storage_t data_ {};
-
- msg_t() = default;
- msg_t(msg_id_t cc_id, msg_id_t id, std::int32_t remain, void const * data, std::size_t size)
- : msg_t<0, AlignSize> {cc_id, id, remain, (data == nullptr) || (size == 0)} {
- if (this->storage_) {
- if (data != nullptr) {
- // copy storage-id
- *reinterpret_cast(&data_) =
- *static_cast(data);
- }
- }
- else std::memcpy(&data_, data, size);
- }
-};
-
-template
-ipc::buff_t make_cache(T& data, std::size_t size) {
- auto ptr = ipc::mem::alloc(size);
- std::memcpy(ptr, &data, (ipc::detail::min)(sizeof(data), size));
- return { ptr, size, ipc::mem::free };
-}
-
-struct cache_t {
- std::size_t fill_;
- ipc::buff_t buff_;
-
- cache_t(std::size_t f, ipc::buff_t && b)
- : fill_(f), buff_(std::move(b))
- {}
-
- void append(void const * data, std::size_t size) {
- if (fill_ >= buff_.size() || data == nullptr || size == 0) return;
- auto new_fill = (ipc::detail::min)(fill_ + size, buff_.size());
- std::memcpy(static_cast(buff_.data()) + fill_, data, new_fill - fill_);
- fill_ = new_fill;
- }
-};
-
-auto cc_acc() {
- static ipc::shm::handle acc_h("__CA_CONN__", sizeof(acc_t));
- return static_cast(acc_h.get());
-}
-
-IPC_CONSTEXPR_ std::size_t align_chunk_size(std::size_t size) noexcept {
- return (((size - 1) / ipc::large_msg_align) + 1) * ipc::large_msg_align;
-}
-
-IPC_CONSTEXPR_ std::size_t calc_chunk_size(std::size_t size) noexcept {
- return ipc::make_align(alignof(std::max_align_t), align_chunk_size(
- ipc::make_align(alignof(std::max_align_t), sizeof(std::atomic)) + size));
-}
-
-struct chunk_t {
- std::atomic &conns() noexcept {
- return *reinterpret_cast *>(this);
- }
-
- void *data() noexcept {
- return reinterpret_cast(this)
- + ipc::make_align(alignof(std::max_align_t), sizeof(std::atomic));
- }
-};
-
-struct chunk_info_t {
- ipc::id_pool<> pool_;
- ipc::spin_lock lock_;
-
- IPC_CONSTEXPR_ static std::size_t chunks_mem_size(std::size_t chunk_size) noexcept {
- return ipc::id_pool<>::max_count * chunk_size;
- }
-
- ipc::byte_t *chunks_mem() noexcept {
- return reinterpret_cast(this + 1);
- }
-
- chunk_t *at(std::size_t chunk_size, ipc::storage_id_t id) noexcept {
- if (id < 0) return nullptr;
- return reinterpret_cast(chunks_mem() + (chunk_size * id));
- }
-};
-
-auto& chunk_storages() {
- class chunk_handle_t {
- ipc::shm::handle handle_;
-
- public:
- chunk_info_t *get_info(std::size_t chunk_size) {
- if (!handle_.valid() &&
- !handle_.acquire( ("__CHUNK_INFO__" + ipc::to_string(chunk_size)).c_str(),
- sizeof(chunk_info_t) + chunk_info_t::chunks_mem_size(chunk_size) )) {
- ipc::error("[chunk_storages] chunk_shm.id_info_.acquire failed: chunk_size = %zd\n", chunk_size);
- return nullptr;
- }
- auto info = static_cast(handle_.get());
- if (info == nullptr) {
- ipc::error("[chunk_storages] chunk_shm.id_info_.get failed: chunk_size = %zd\n", chunk_size);
- return nullptr;
- }
- return info;
- }
- };
- static ipc::map chunk_hs;
- return chunk_hs;
-}
-
-chunk_info_t *chunk_storage_info(std::size_t chunk_size) {
- auto &storages = chunk_storages();
- std::decay_t::iterator it;
- {
- static ipc::rw_lock lock;
- IPC_UNUSED_ std::shared_lock guard {lock};
- if ((it = storages.find(chunk_size)) == storages.end()) {
- using chunk_handle_t = std::decay_t::value_type::second_type;
- guard.unlock();
- IPC_UNUSED_ std::lock_guard guard {lock};
- it = storages.emplace(chunk_size, chunk_handle_t{}).first;
- }
- }
- return it->second.get_info(chunk_size);
-}
-
-std::pair acquire_storage(std::size_t size, ipc::circ::cc_t conns) {
- std::size_t chunk_size = calc_chunk_size(size);
- auto info = chunk_storage_info(chunk_size);
- if (info == nullptr) return {};
-
- info->lock_.lock();
- info->pool_.prepare();
- // got an unique id
- auto id = info->pool_.acquire();
- info->lock_.unlock();
-
- auto chunk = info->at(chunk_size, id);
- if (chunk == nullptr) return {};
- chunk->conns().store(conns, std::memory_order_relaxed);
- return { id, chunk->data() };
-}
-
-void *find_storage(ipc::storage_id_t id, std::size_t size) {
- if (id < 0) {
- ipc::error("[find_storage] id is invalid: id = %ld, size = %zd\n", (long)id, size);
- return nullptr;
- }
- std::size_t chunk_size = calc_chunk_size(size);
- auto info = chunk_storage_info(chunk_size);
- if (info == nullptr) return nullptr;
- return info->at(chunk_size, id)->data();
-}
-
-void release_storage(ipc::storage_id_t id, std::size_t size) {
- if (id < 0) {
- ipc::error("[release_storage] id is invalid: id = %ld, size = %zd\n", (long)id, size);
- return;
- }
- std::size_t chunk_size = calc_chunk_size(size);
- auto info = chunk_storage_info(chunk_size);
- if (info == nullptr) return;
- info->lock_.lock();
- info->pool_.release(id);
- info->lock_.unlock();
-}
-
-template
-bool sub_rc(ipc::wr