From 172f9e220be5c7c82c005d6afd78f8f0bb9c727a Mon Sep 17 00:00:00 2001 From: binary-husky Date: Sat, 5 Oct 2024 16:51:08 +0000 Subject: [PATCH 1/6] version 3.90 --- .../build-with-all-capacity-beta.yml | 44 -------------- docs/GithubAction+AllCapacityBeta | 57 ------------------- version | 4 +- 3 files changed, 2 insertions(+), 103 deletions(-) delete mode 100644 .github/workflows/build-with-all-capacity-beta.yml delete mode 100644 docs/GithubAction+AllCapacityBeta diff --git a/.github/workflows/build-with-all-capacity-beta.yml b/.github/workflows/build-with-all-capacity-beta.yml deleted file mode 100644 index 5a2a1a54..00000000 --- a/.github/workflows/build-with-all-capacity-beta.yml +++ /dev/null @@ -1,44 +0,0 @@ -# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images#publishing-images-to-github-packages -name: build-with-all-capacity-beta - -on: - push: - branches: - - 'master' - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }}_with_all_capacity_beta - -jobs: - build-and-push-image: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Build and push Docker image - uses: docker/build-push-action@v4 - with: - context: . - push: true - file: docs/GithubAction+AllCapacityBeta - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/docs/GithubAction+AllCapacityBeta b/docs/GithubAction+AllCapacityBeta deleted file mode 100644 index e942926c..00000000 --- a/docs/GithubAction+AllCapacityBeta +++ /dev/null @@ -1,57 +0,0 @@ -# docker build -t gpt-academic-all-capacity -f docs/GithubAction+AllCapacity --network=host --build-arg http_proxy=http://localhost:10881 --build-arg https_proxy=http://localhost:10881 . -# docker build -t gpt-academic-all-capacity -f docs/GithubAction+AllCapacityBeta --network=host . -# docker run -it --net=host gpt-academic-all-capacity bash - -# 从NVIDIA源，从而支持显卡（检查宿主的nvidia-smi中的cuda版本必须>=11.3） -FROM fuqingxu/11.3.1-runtime-ubuntu20.04-with-texlive:latest - -# edge-tts需要的依赖，某些pip包所需的依赖 -RUN apt update && apt install ffmpeg build-essential -y - -# use python3 as the system default python -WORKDIR /gpt -RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.8 - -# # 非必要步骤，更换pip源 （以下三行，可以删除） -# RUN echo '[global]' > /etc/pip.conf && \ -# echo 'index-url = https://mirrors.aliyun.com/pypi/simple/' >> /etc/pip.conf && \ -# echo 'trusted-host = mirrors.aliyun.com' >> /etc/pip.conf - -# 下载pytorch -RUN python3 -m pip install torch torchvision --extra-index-url https://download.pytorch.org/whl/cu113 -# 准备pip依赖 -RUN python3 -m pip install openai numpy arxiv rich -RUN python3 -m pip install colorama Markdown pygments pymupdf -RUN python3 -m pip install python-docx moviepy pdfminer -RUN python3 -m pip install zh_langchain==0.2.1 pypinyin -RUN python3 -m pip install rarfile py7zr -RUN python3 -m pip install aliyun-python-sdk-core==2.13.3 pyOpenSSL webrtcvad scipy git+https://github.com/aliyun/alibabacloud-nls-python-sdk.git -# 下载分支 -WORKDIR /gpt -RUN git clone --depth=1 https://github.com/binary-husky/gpt_academic.git -WORKDIR /gpt/gpt_academic -RUN git clone --depth=1 https://github.com/OpenLMLab/MOSS.git request_llms/moss - -RUN python3 -m pip install -r requirements.txt -RUN python3 -m pip install -r request_llms/requirements_moss.txt -RUN python3 -m pip install -r request_llms/requirements_qwen.txt -RUN python3 -m pip install -r request_llms/requirements_chatglm.txt -RUN python3 -m pip install -r request_llms/requirements_newbing.txt -RUN python3 -m pip install nougat-ocr - - -# 预热Tiktoken模块 -RUN python3 -c 'from check_proxy import warm_up_modules; warm_up_modules()' - -# 安装知识库插件的额外依赖 -RUN apt-get update && apt-get install libgl1 -y -RUN pip3 install transformers protobuf langchain sentence-transformers faiss-cpu nltk beautifulsoup4 bitsandbytes tabulate icetk --upgrade -RUN pip3 install unstructured[all-docs] --upgrade -RUN python3 -c 'from check_proxy import warm_up_vectordb; warm_up_vectordb()' -RUN rm -rf /usr/local/lib/python3.8/dist-packages/tests - - -# COPY .cache /root/.cache -# COPY config_private.py config_private.py -# 启动 -CMD ["python3", "-u", "main.py"] diff --git a/version b/version index d417ea2a..e0e936dd 100644 --- a/version +++ b/version @@ -1,5 +1,5 @@ { - "version": 3.83, + "version": 3.90, "show_feature": true, - "new_feature": "增加欢迎页面 <-> 优化图像生成插件 <-> 添加紫东太初大模型支持 <-> 保留主题选择 <-> 支持更复杂的插件框架 <-> 上传文件时显示进度条" + "new_feature": "增加RAG组件 <-> 升级多合一主提交键" } From 9ac450cfb6240f85d402d0e757940d84cff9b79f Mon Sep 17 00:00:00 2001 From: binary-husky Date: Sun, 6 Oct 2024 15:02:14 +0000 Subject: [PATCH 2/6] =?UTF-8?q?=E7=B4=A7=E6=80=A5=E4=BF=AE=E5=A4=8D=20fix?= =?UTF-8?q?=20httpx=20breaking=20bad=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 25b9da94..7a3d9f86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ https://public.agent-matrix.com/publish/gradio-3.32.10-py3-none-any.whl fastapi==0.110 gradio-client==0.8 pypdf2==2.12.1 +httpx<=0.25.2 zhipuai==2.0.1 tiktoken>=0.3.3 requests[socks] From da4a5efc491601a058f626263ab4eb650210b32e Mon Sep 17 00:00:00 2001 From: binary-husky Date: Sun, 6 Oct 2024 16:26:26 +0000 Subject: [PATCH 3/6] lazy load llama-index lib --- crazy_functions/Rag_Interface.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/crazy_functions/Rag_Interface.py b/crazy_functions/Rag_Interface.py index d83d8ca5..1bc740ad 100644 --- a/crazy_functions/Rag_Interface.py +++ b/crazy_functions/Rag_Interface.py @@ -2,20 +2,7 @@ from toolbox import CatchException, update_ui, get_conf, get_log_folder, update_ from crazy_functions.crazy_utils import input_clipping from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive -VECTOR_STORE_TYPE = "Milvus" - -if VECTOR_STORE_TYPE == "Milvus": - try: - from crazy_functions.rag_fns.milvus_worker import MilvusRagWorker as LlamaIndexRagWorker - except: - VECTOR_STORE_TYPE = "Simple" - -if VECTOR_STORE_TYPE == "Simple": - from crazy_functions.rag_fns.llama_index_worker import LlamaIndexRagWorker - - RAG_WORKER_REGISTER = {} - MAX_HISTORY_ROUND = 5 MAX_CONTEXT_TOKEN_LIMIT = 4096 REMEMBER_PREVIEW = 1000 @@ -23,6 +10,16 @@ REMEMBER_PREVIEW = 1000 @CatchException def Rag问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request): + # import vector store lib + VECTOR_STORE_TYPE = "Milvus" + if VECTOR_STORE_TYPE == "Milvus": + try: + from crazy_functions.rag_fns.milvus_worker import MilvusRagWorker as LlamaIndexRagWorker + except: + VECTOR_STORE_TYPE = "Simple" + if VECTOR_STORE_TYPE == "Simple": + from crazy_functions.rag_fns.llama_index_worker import LlamaIndexRagWorker + # 1. we retrieve rag worker from global context user_name = chatbot.get_user() checkpoint_dir = get_log_folder(user_name, plugin_name='experimental_rag') From 12c9ab1e33d13ff01bc758018ac7a8747a0eaeee Mon Sep 17 00:00:00 2001 From: binary-husky <96192199+binary-husky@users.noreply.github.com> Date: Thu, 10 Oct 2024 12:02:12 +0800 Subject: [PATCH 4/6] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 97da208f..430583fa 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ > [!IMPORTANT] -> 2024.6.1: 版本3.80加入插件二级菜单功能（详见wiki） +> 2024.10.10: 突发停电，紧急恢复了提供whl包的文件服务器 +> 2024.10.8: 版本3.90加入对llama-index的初步支持，版本3.80加入插件二级菜单功能（详见wiki） > 2024.5.1: 加入Doc2x翻译PDF论文的功能，[查看详情](https://github.com/binary-husky/gpt_academic/wiki/Doc2x) > 2024.3.11: 全力支持Qwen、GLM、DeepseekCoder等中文大语言模型！ SoVits语音克隆模块，[查看详情](https://www.bilibili.com/video/BV1Rp421S7tF/) > 2024.1.17: 安装依赖时，请选择`requirements.txt`中**指定的版本**。 安装命令：`pip install -r requirements.txt`。本项目完全开源免费，您可通过订阅[在线服务](https://github.com/binary-husky/gpt_academic/wiki/online)的方式鼓励本项目的发展。 From 679352d896712a11cc32cc9ecdf0c69fa049c7cc Mon Sep 17 00:00:00 2001 From: binary-husky <96192199+binary-husky@users.noreply.github.com> Date: Thu, 10 Oct 2024 13:38:35 +0800 Subject: [PATCH 5/6] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 430583fa..2b8ffaa5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ > [!IMPORTANT] -> 2024.10.10: 突发停电，紧急恢复了提供whl包的文件服务器 +> 2024.10.10: 突发停电，紧急恢复了提供[whl包](https://drive.google.com/file/d/19U_hsLoMrjOlQSzYS3pzWX9fTzyusArP/view?usp=sharing)的文件服务器 > 2024.10.8: 版本3.90加入对llama-index的初步支持，版本3.80加入插件二级菜单功能（详见wiki） > 2024.5.1: 加入Doc2x翻译PDF论文的功能，[查看详情](https://github.com/binary-husky/gpt_academic/wiki/Doc2x) > 2024.3.11: 全力支持Qwen、GLM、DeepseekCoder等中文大语言模型！ SoVits语音克隆模块，[查看详情](https://www.bilibili.com/video/BV1Rp421S7tF/) From f695d7f1da1bf7a033fb256e4d370e59c0f83d39 Mon Sep 17 00:00:00 2001 From: binary-husky Date: Fri, 11 Oct 2024 13:32:57 +0000 Subject: [PATCH 6/6] test math anim generation --- tests/test_anim_gen.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tests/test_anim_gen.py diff --git a/tests/test_anim_gen.py b/tests/test_anim_gen.py new file mode 100644 index 00000000..0084f841 --- /dev/null +++ b/tests/test_anim_gen.py @@ -0,0 +1,12 @@ +""" +对项目中的各个插件进行测试。运行方法：直接运行 python tests/test_plugins.py +""" + +import init_test +import os, sys + + +if __name__ == "__main__": + from test_utils import plugin_test + + plugin_test(plugin='crazy_functions.数学动画生成manim->动画生成', main_input="A point moving along function culve y=sin(x), starting from x=0 and stop at x=4*\pi.")