solve the pdf concate error

.github/workflows/build-with-all-capacity-beta.yml

@@ -1,14 +1,14 @@
 # https://docs.github.com/en/actions/publishing-packages/publishing-docker-images#publishing-images-to-github-packages
-name: build-with-latex-arm
+name: build-with-all-capacity-beta
 
 on:
   push:
     branches:
-      - "master"
+      - 'master'
 
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}_with_latex_arm
+  IMAGE_NAME: ${{ github.repository }}_with_all_capacity_beta
 
 jobs:
   build-and-push-image:
@@ -18,17 +18,11 @@ jobs:
       packages: write
 
     steps:
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -41,11 +35,10 @@ jobs:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v4
         with:
           context: .
           push: true
-          platforms: linux/arm64
+          file: docs/GithubAction+AllCapacityBeta
-          file: docs/GithubAction+NoLocal+Latex+Arm
           tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/build-with-jittorllms.yml

@@ -0,0 +1,44 @@
+# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images#publishing-images-to-github-packages
+name: build-with-jittorllms
+
+on:
+  push:
+    branches:
+      - 'master'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}_jittorllms
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          file: docs/GithubAction+JittorLLMs
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

README.md

@@ -1,6 +1,5 @@
 > [!IMPORTANT]
-> 2024.10.10: 突发停电，紧急恢复了提供[whl包](https://drive.google.com/file/d/19U_hsLoMrjOlQSzYS3pzWX9fTzyusArP/view?usp=sharing)的文件服务器  
+> 2024.6.1: 版本3.80加入插件二级菜单功能（详见wiki）  
-> 2024.10.8: 版本3.90加入对llama-index的初步支持，版本3.80加入插件二级菜单功能（详见wiki）  
 > 2024.5.1: 加入Doc2x翻译PDF论文的功能，[查看详情](https://github.com/binary-husky/gpt_academic/wiki/Doc2x)  
 > 2024.3.11: 全力支持Qwen、GLM、DeepseekCoder等中文大语言模型！ SoVits语音克隆模块，[查看详情](https://www.bilibili.com/video/BV1Rp421S7tF/) 
 > 2024.1.17: 安装依赖时，请选择`requirements.txt`中**指定的版本**。 安装命令：`pip install -r requirements.txt`。本项目完全开源免费，您可通过订阅[在线服务](https://github.com/binary-husky/gpt_academic/wiki/online)的方式鼓励本项目的发展。

crazy_functional.py

@@ -6,6 +6,7 @@ from loguru import logger
 def get_crazy_functions():
     from crazy_functions.读文章写摘要 import 读文章写摘要
     from crazy_functions.生成函数注释 import 批量生成函数注释
+    from crazy_functions.Rag_Interface import Rag问答
     from crazy_functions.SourceCode_Analyse import 解析项目本身
     from crazy_functions.SourceCode_Analyse import 解析一个Python项目
     from crazy_functions.SourceCode_Analyse import 解析一个Matlab项目
@@ -51,6 +52,13 @@ def get_crazy_functions():
     from crazy_functions.SourceCode_Comment import 注释Python项目
 
     function_plugins = {
+        "Rag智能召回": {
+            "Group": "对话",
+            "Color": "stop",
+            "AsButton": False,
+            "Info": "将问答数据记录到向量库中，作为长期参考。",
+            "Function": HotReload(Rag问答),
+        },
         "虚空终端": {
             "Group": "对话|编程|学术|智能体",
             "Color": "stop",
@@ -699,31 +707,6 @@ def get_crazy_functions():
         logger.error(trimmed_format_exc())
         logger.error("Load function plugin failed")
 
-    try:
-        from crazy_functions.Rag_Interface import Rag问答
-
-        function_plugins.update(
-            {
-                "Rag智能召回": {
-                    "Group": "对话",
-                    "Color": "stop",
-                    "AsButton": False,
-                    "Info": "将问答数据记录到向量库中，作为长期参考。",
-                    "Function": HotReload(Rag问答),
-                },
-            }
-        )
-    except:
-        logger.error(trimmed_format_exc())
-        logger.error("Load function plugin failed")
-
-
-    
-
-
-
-
-
     # try:
     #     from crazy_functions.高级功能函数模板 import 测试图表渲染
     #     function_plugins.update({

crazy_functions/Latex_Function.py

@@ -3,7 +3,7 @@ from toolbox import CatchException, report_exception, update_ui_lastest_msg, zip
 from functools import partial
 from loguru import logger
 
-import glob, os, requests, time, json, tarfile, threading
+import glob, os, requests, time, json, tarfile
 
 pj = os.path.join
 ARXIV_CACHE_DIR = get_conf("ARXIV_CACHE_DIR")
@@ -138,43 +138,25 @@ def arxiv_download(chatbot, history, txt, allow_cache=True):
     cached_translation_pdf = check_cached_translation_pdf(arxiv_id)
     if cached_translation_pdf and allow_cache: return cached_translation_pdf, arxiv_id
 
-    extract_dst = pj(ARXIV_CACHE_DIR, arxiv_id, 'extract')
+    url_tar = url_.replace('/abs/', '/e-print/')
     translation_dir = pj(ARXIV_CACHE_DIR, arxiv_id, 'e-print')
-    dst = pj(translation_dir, arxiv_id + '.tar')
+    extract_dst = pj(ARXIV_CACHE_DIR, arxiv_id, 'extract')
     os.makedirs(translation_dir, exist_ok=True)
+
     # <-------------- download arxiv source file ------------->
-
+    dst = pj(translation_dir, arxiv_id + '.tar')
-    def fix_url_and_download():
+    if os.path.exists(dst):
-        # for url_tar in [url_.replace('/abs/', '/e-print/'), url_.replace('/abs/', '/src/')]:
+        yield from update_ui_lastest_msg("调用缓存", chatbot=chatbot, history=history)  # 刷新界面
-        for url_tar in [url_.replace('/abs/', '/src/'), url_.replace('/abs/', '/e-print/')]:
-            proxies = get_conf('proxies')
-            r = requests.get(url_tar, proxies=proxies)
-            if r.status_code == 200:
-                with open(dst, 'wb+') as f:
-                    f.write(r.content)
-                return True
-        return False
-
-    if os.path.exists(dst) and allow_cache:
-        yield from update_ui_lastest_msg(f"调用缓存 {arxiv_id}", chatbot=chatbot, history=history)  # 刷新界面
-        success = True
     else:
-        yield from update_ui_lastest_msg(f"开始下载 {arxiv_id}", chatbot=chatbot, history=history)  # 刷新界面
+        yield from update_ui_lastest_msg("开始下载", chatbot=chatbot, history=history)  # 刷新界面
-        success = fix_url_and_download()
+        proxies = get_conf('proxies')
-        yield from update_ui_lastest_msg(f"下载完成 {arxiv_id}", chatbot=chatbot, history=history)  # 刷新界面
+        r = requests.get(url_tar, proxies=proxies)
-
+        with open(dst, 'wb+') as f:
-
+            f.write(r.content)
-    if not success:
-        yield from update_ui_lastest_msg(f"下载失败 {arxiv_id}", chatbot=chatbot, history=history)
-        raise tarfile.ReadError(f"论文下载失败 {arxiv_id}")
-
     # <-------------- extract file ------------->
+    yield from update_ui_lastest_msg("下载完成", chatbot=chatbot, history=history)  # 刷新界面
     from toolbox import extract_archive
-    try:
+    extract_archive(file_path=dst, dest_dir=extract_dst)
-        extract_archive(file_path=dst, dest_dir=extract_dst)
-    except tarfile.ReadError:
-        os.remove(dst)
-        raise tarfile.ReadError(f"论文下载失败")
     return extract_dst, arxiv_id
 
 
@@ -338,17 +320,11 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot,
     # <-------------- more requirements ------------->
     if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg")
     more_req = plugin_kwargs.get("advanced_arg", "")
-
+    no_cache = more_req.startswith("--no-cache")
-    no_cache = ("--no-cache" in more_req)
+    if no_cache: more_req.lstrip("--no-cache")
-    if no_cache: more_req = more_req.replace("--no-cache", "").strip()
-
-    allow_gptac_cloud_io = ("--allow-cloudio" in more_req)  # 从云端下载翻译结果，以及上传翻译结果到云端
-    if allow_gptac_cloud_io: more_req = more_req.replace("--allow-cloudio", "").strip()
-
     allow_cache = not no_cache
     _switch_prompt_ = partial(switch_prompt, more_requirement=more_req)
 
-
     # <-------------- check deps ------------->
     try:
         import glob, os, time, subprocess
@@ -375,20 +351,6 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot,
         yield from update_ui(chatbot=chatbot, history=history)  # 刷新界面
         return
 
-    # #################################################################
-    if allow_gptac_cloud_io and arxiv_id:
-        # 访问 GPTAC学术云，查询云端是否存在该论文的翻译版本
-        from crazy_functions.latex_fns.latex_actions import check_gptac_cloud
-        success, downloaded = check_gptac_cloud(arxiv_id, chatbot)
-        if success:
-            chatbot.append([
-                f"检测到GPTAC云端存在翻译版本, 如果不满意翻译结果, 请禁用云端分享, 然后重新执行。", 
-                None
-            ])
-            yield from update_ui(chatbot=chatbot, history=history)
-            return
-    #################################################################
-
     if os.path.exists(txt):
         project_folder = txt
     else:
@@ -426,21 +388,14 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot,
     # <-------------- zip PDF ------------->
     zip_res = zip_result(project_folder)
     if success:
-        if allow_gptac_cloud_io and arxiv_id:
-            # 如果用户允许，我们将翻译好的arxiv论文PDF上传到GPTAC学术云
-            from crazy_functions.latex_fns.latex_actions import upload_to_gptac_cloud_if_user_allow
-            threading.Thread(target=upload_to_gptac_cloud_if_user_allow, 
-                args=(chatbot, arxiv_id), daemon=True).start()
-
         chatbot.append((f"成功啦", '请查收结果（压缩包）...'))
-        yield from update_ui(chatbot=chatbot, history=history)
+        yield from update_ui(chatbot=chatbot, history=history);
         time.sleep(1)  # 刷新界面
         promote_file_to_downloadzone(file=zip_res, chatbot=chatbot)
-
     else:
         chatbot.append((f"失败了",
                         '虽然PDF生成失败了, 但请查收结果（压缩包）, 内含已经翻译的Tex文档, 您可以到Github Issue区, 用该压缩包进行反馈。如系统是Linux，请检查系统字体（见Github wiki） ...'))
-        yield from update_ui(chatbot=chatbot, history=history)
+        yield from update_ui(chatbot=chatbot, history=history);
         time.sleep(1)  # 刷新界面
         promote_file_to_downloadzone(file=zip_res, chatbot=chatbot)
 

crazy_functions/Latex_Function_Wrap.py

@@ -30,9 +30,6 @@ class Arxiv_Localize(GptAcademicPluginTemplate):
                             default_value="", type="string").model_dump_json(), # 高级参数输入区，自动同步
             "allow_cache":
                 ArgProperty(title="是否允许从缓存中调取结果", options=["允许缓存", "从头执行"], default_value="允许缓存", description="无", type="dropdown").model_dump_json(),
-            "allow_cloudio":
-                ArgProperty(title="是否允许向GPTAC学术云共享翻译结果", options=["允许", "禁止"], default_value="禁止", description="人人为我，我为人人", type="dropdown").model_dump_json(),
-
         }
         return gui_definition
 
@@ -41,14 +38,9 @@ class Arxiv_Localize(GptAcademicPluginTemplate):
         执行插件
         """
         allow_cache = plugin_kwargs["allow_cache"]
-        allow_cloudio = plugin_kwargs["allow_cloudio"]
         advanced_arg = plugin_kwargs["advanced_arg"]
 
         if allow_cache == "从头执行": plugin_kwargs["advanced_arg"] = "--no-cache " + plugin_kwargs["advanced_arg"]
-
-        # 从云端下载翻译结果，以及上传翻译结果到云端；人人为我，我为人人。
-        if allow_cloudio == "允许": plugin_kwargs["advanced_arg"] = "--allow-cloudio " + plugin_kwargs["advanced_arg"]
-
         yield from Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request)
 
 

crazy_functions/Rag_Interface.py

@@ -1,108 +1,53 @@
-import os,glob
-from typing import List
-
-from shared_utils.fastapi_server import validate_path_safety
-
-from toolbox import report_exception
 from toolbox import CatchException, update_ui, get_conf, get_log_folder, update_ui_lastest_msg
-from shared_utils.fastapi_server import validate_path_safety
 from crazy_functions.crazy_utils import input_clipping
 from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
 
+VECTOR_STORE_TYPE = "Milvus"
+
+if VECTOR_STORE_TYPE == "Milvus":
+    try:
+        from crazy_functions.rag_fns.milvus_worker import MilvusRagWorker as LlamaIndexRagWorker
+    except:
+        VECTOR_STORE_TYPE = "Simple"
+
+if VECTOR_STORE_TYPE == "Simple":
+    from crazy_functions.rag_fns.llama_index_worker import LlamaIndexRagWorker
+
+
 RAG_WORKER_REGISTER = {}
+
 MAX_HISTORY_ROUND = 5
 MAX_CONTEXT_TOKEN_LIMIT = 4096
 REMEMBER_PREVIEW = 1000
 
-@CatchException
-def handle_document_upload(files: List[str], llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request, rag_worker):
-    """
-    Handles document uploads by extracting text and adding it to the vector store.
-    """
-    from llama_index.core import Document
-    from crazy_functions.rag_fns.rag_file_support import extract_text, supports_format
-    user_name = chatbot.get_user()
-    checkpoint_dir = get_log_folder(user_name, plugin_name='experimental_rag')
-
-    for file_path in files:
-        try:
-            validate_path_safety(file_path, user_name)
-            text = extract_text(file_path)
-            if text is None:
-                chatbot.append(
-                    [f"上传文件: {os.path.basename(file_path)}", f"文件解析失败，无法提取文本内容，请更换文件。失败原因可能为：1.文档格式过于复杂；2. 不支持的文件格式，支持的文件格式后缀有:" + ", ".join(supports_format)])
-            else:
-                chatbot.append(
-                    [f"上传文件: {os.path.basename(file_path)}", f"上传文件前50个字符为:{text[:50]}。"])
-                document = Document(text=text, metadata={"source": file_path})
-                rag_worker.add_documents_to_vector_store([document])
-                chatbot.append([f"上传文件: {os.path.basename(file_path)}", "文件已成功添加到知识库。"])
-        except Exception as e:
-            report_exception(chatbot, history, a=f"处理文件: {file_path}", b=str(e))
-
-    yield from update_ui(chatbot=chatbot, history=history)  # 刷新界面
-
-
-
-# Main Q&A function with document upload support
 @CatchException
 def Rag问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request):
 
-    # import vector store lib
-    VECTOR_STORE_TYPE = "Milvus"
-    if VECTOR_STORE_TYPE == "Milvus":
-        try:
-            from crazy_functions.rag_fns.milvus_worker import MilvusRagWorker as LlamaIndexRagWorker
-        except:
-            VECTOR_STORE_TYPE = "Simple"
-    if VECTOR_STORE_TYPE == "Simple":
-        from crazy_functions.rag_fns.llama_index_worker import LlamaIndexRagWorker
-
     # 1. we retrieve rag worker from global context
     user_name = chatbot.get_user()
     checkpoint_dir = get_log_folder(user_name, plugin_name='experimental_rag')
-
     if user_name in RAG_WORKER_REGISTER:
         rag_worker = RAG_WORKER_REGISTER[user_name]
     else:
         rag_worker = RAG_WORKER_REGISTER[user_name] = LlamaIndexRagWorker(
-            user_name,
+            user_name, 
-            llm_kwargs,
+            llm_kwargs, 
-            checkpoint_dir=checkpoint_dir,
+            checkpoint_dir=checkpoint_dir, 
-            auto_load_checkpoint=True
+            auto_load_checkpoint=True)
-        )
-
     current_context = f"{VECTOR_STORE_TYPE} @ {checkpoint_dir}"
     tip = "提示：输入“清空向量数据库”可以清空RAG向量数据库"
-
+    if txt == "清空向量数据库":
-    # 2. Handle special commands
-    if os.path.exists(txt) and os.path.isdir(txt):
-        project_folder = txt
-        validate_path_safety(project_folder, chatbot.get_user())
-        # Extract file paths from the user input
-        # Assuming the user inputs file paths separated by commas after the command
-        file_paths =  [f for f in glob.glob(f'{project_folder}/**/*', recursive=True)]
-        chatbot.append([txt, f'正在处理上传的文档 ({current_context}) ...'])
-        yield from update_ui(chatbot=chatbot, history=history)  # 刷新界面
-
-        yield from handle_document_upload(file_paths, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request, rag_worker)
-        return
-
-    elif txt == "清空向量数据库":
         chatbot.append([txt, f'正在清空 ({current_context}) ...'])
-        yield from update_ui(chatbot=chatbot, history=history)  # 刷新界面
+        yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
-        rag_worker.purge_vector_store()
+        rag_worker.purge()
-        yield from update_ui_lastest_msg('已清空', chatbot, history, delay=0)  # 刷新界面
+        yield from update_ui_lastest_msg('已清空', chatbot, history, delay=0) # 刷新界面
         return
 
-    else:
-        report_exception(chatbot, history, a=f"上传文件路径错误: {txt}", b="请检查并提供正确路径。")
-
-    # 3. Normal Q&A processing
     chatbot.append([txt, f'正在召回知识 ({current_context}) ...'])
-    yield from update_ui(chatbot=chatbot, history=history)  # 刷新界面
+    yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
 
-    # 4. Clip history to reduce token consumption
+    # 2. clip history to reduce token consumption
+    #   2-1. reduce chat round
     txt_origin = txt
 
     if len(history) > MAX_HISTORY_ROUND * 2:
@@ -110,47 +55,41 @@ def Rag问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, u
     txt_clip, history, flags = input_clipping(txt, history, max_token_limit=MAX_CONTEXT_TOKEN_LIMIT, return_clip_flags=True)
     input_is_clipped_flag = (flags["original_input_len"] != flags["clipped_input_len"])
 
-    # 5. If input is clipped, add input to vector store before retrieve
+    #   2-2. if input is clipped, add input to vector store before retrieve
     if input_is_clipped_flag:
-        yield from update_ui_lastest_msg('检测到长输入, 正在向量化 ...', chatbot, history, delay=0)  # 刷新界面
+        yield from update_ui_lastest_msg('检测到长输入, 正在向量化 ...', chatbot, history, delay=0) # 刷新界面
-        # Save input to vector store
+        # save input to vector store
         rag_worker.add_text_to_vector_store(txt_origin)
-        yield from update_ui_lastest_msg('向量化完成 ...', chatbot, history, delay=0)  # 刷新界面
+        yield from update_ui_lastest_msg('向量化完成 ...', chatbot, history, delay=0) # 刷新界面
-
         if len(txt_origin) > REMEMBER_PREVIEW:
-            HALF = REMEMBER_PREVIEW // 2
+            HALF = REMEMBER_PREVIEW//2
             i_say_to_remember = txt[:HALF] + f" ...\n...(省略{len(txt_origin)-REMEMBER_PREVIEW}字)...\n... " + txt[-HALF:]
             if (flags["original_input_len"] - flags["clipped_input_len"]) > HALF:
-                txt_clip = txt_clip + f" ...\n...(省略{len(txt_origin)-len(txt_clip)-HALF}字)...\n... " + txt[-HALF:]
+                txt_clip = txt_clip  + f" ...\n...(省略{len(txt_origin)-len(txt_clip)-HALF}字)...\n... " + txt[-HALF:]
+            else:
+                pass
+            i_say = txt_clip
         else:
             i_say_to_remember = i_say = txt_clip
     else:
         i_say_to_remember = i_say = txt_clip
 
-    # 6. Search vector store and build prompts
+    # 3. we search vector store and build prompts
     nodes = rag_worker.retrieve_from_store_with_query(i_say)
     prompt = rag_worker.build_prompt(query=i_say, nodes=nodes)
-    # 7. Query language model
-    if len(chatbot) != 0:
-        chatbot.pop(-1)  # Pop temp chat, because we are going to add them again inside `request_gpt_model_in_new_thread_with_ui_alive`
 
+    # 4. it is time to query llms
+    if len(chatbot) != 0: chatbot.pop(-1) # pop temp chat, because we are going to add them again inside `request_gpt_model_in_new_thread_with_ui_alive`
     model_say = yield from request_gpt_model_in_new_thread_with_ui_alive(
-        inputs=prompt,
+        inputs=prompt, inputs_show_user=i_say,
-        inputs_show_user=i_say,
+        llm_kwargs=llm_kwargs, chatbot=chatbot, history=history,
-        llm_kwargs=llm_kwargs,
-        chatbot=chatbot,
-        history=history,
         sys_prompt=system_prompt,
         retry_times_at_unknown_error=0
     )
 
-    # 8. Remember Q&A
+    # 5. remember what has been asked / answered
-    yield from update_ui_lastest_msg(
+    yield from update_ui_lastest_msg(model_say + '</br></br>' + f'对话记忆中, 请稍等 ({current_context}) ...', chatbot, history, delay=0.5) # 刷新界面
-        model_say + '</br></br>' + f'对话记忆中, 请稍等 ({current_context}) ...',
-        chatbot, history, delay=0.5
-    )
     rag_worker.remember_qa(i_say_to_remember, model_say)
     history.extend([i_say, model_say])
 
-    # 9. Final UI Update
+    yield from update_ui_lastest_msg(model_say, chatbot, history, delay=0, msg=tip) # 刷新界面
-    yield from update_ui_lastest_msg(model_say, chatbot, history, delay=0, msg=tip)

crazy_functions/latex_fns/latex_actions.py

@@ -3,7 +3,7 @@ import re
 import shutil
 import numpy as np
 from loguru import logger
-from toolbox import update_ui, update_ui_lastest_msg, get_log_folder, gen_time_str
+from toolbox import update_ui, update_ui_lastest_msg, get_log_folder
 from toolbox import get_conf, promote_file_to_downloadzone
 from crazy_functions.latex_fns.latex_toolbox import PRESERVE, TRANSFORM
 from crazy_functions.latex_fns.latex_toolbox import set_forbidden_text, set_forbidden_text_begin_end, set_forbidden_text_careful_brace
@@ -423,9 +423,6 @@ def 编译Latex(chatbot, history, main_file_original, main_file_modified, work_f
                 except Exception as e:
                     logger.error(e)
                     pass
-
-
-
             return True # 成功啦
         else:
             if n_fix>=max_try: break
@@ -471,66 +468,3 @@ def write_html(sp_file_contents, sp_file_result, chatbot, project_folder):
     except:
         from toolbox import trimmed_format_exc
         logger.error('writing html result failed:', trimmed_format_exc())
-
-
-def upload_to_gptac_cloud_if_user_allow(chatbot, arxiv_id):
-    try:
-        # 如果用户允许，我们将arxiv论文PDF上传到GPTAC学术云
-        from toolbox import map_file_to_sha256
-        # 检查是否顺利，如果没有生成预期的文件，则跳过
-        is_result_good = False
-        for file_path in chatbot._cookies.get("files_to_promote", []):
-            if file_path.endswith('translate_zh.pdf'):
-                is_result_good = True
-        if not is_result_good:
-            return
-        # 上传文件
-        for file_path in chatbot._cookies.get("files_to_promote", []):
-            align_name = None
-            # normalized name
-            for name in ['translate_zh.pdf', 'comparison.pdf']:
-                if file_path.endswith(name): align_name = name
-            # if match any align name
-            if align_name:
-                logger.info(f'Uploading to GPTAC cloud as the user has set `allow_cloud_io`: {file_path}')
-                with open(file_path, 'rb') as f:
-                    import requests
-                    url = 'https://cloud-2.agent-matrix.com/upload'
-                    files = {'file': (align_name, f, 'application/octet-stream')}
-                    data = {
-                        'arxiv_id': arxiv_id,
-                        'file_hash': map_file_to_sha256(file_path),
-                    }
-                    resp = requests.post(url=url, files=files, data=data, timeout=30)
-                logger.info(f'Uploading terminate ({resp.status_code})`: {file_path}')
-    except:
-        # 如果上传失败，不会中断程序，因为这是次要功能
-        pass
-
-def check_gptac_cloud(arxiv_id, chatbot):
-    import requests
-    success = False
-    downloaded = []
-    try:
-        for pdf_target in ['translate_zh.pdf', 'comparison.pdf']:
-            url = 'https://cloud-2.agent-matrix.com/paper_exist'
-            data = {
-                'arxiv_id': arxiv_id,
-                'name': pdf_target,
-            }
-            resp = requests.post(url=url, data=data)
-            cache_hit_result = resp.text.strip('"')
-            if cache_hit_result.startswith("http"):
-                url = cache_hit_result
-                logger.info(f'Downloading from GPTAC cloud: {url}')
-                resp = requests.get(url=url, timeout=30)
-                target = os.path.join(get_log_folder(plugin_name='gptac_cloud'), gen_time_str(), pdf_target)
-                os.makedirs(os.path.dirname(target), exist_ok=True)
-                with open(target, 'wb') as f:
-                    f.write(resp.content)
-                new_path = promote_file_to_downloadzone(target, chatbot=chatbot)
-                success = True
-                downloaded.append(new_path)
-    except:
-        pass
-    return success, downloaded

crazy_functions/latex_fns/latex_toolbox.py

@@ -644,17 +644,8 @@ def run_in_subprocess(func):
 
 
 def _merge_pdfs(pdf1_path, pdf2_path, output_path):
-    try:
-        logger.info("Merging PDFs using _merge_pdfs_ng")
-        _merge_pdfs_ng(pdf1_path, pdf2_path, output_path)
-    except:
-        logger.info("Merging PDFs using _merge_pdfs_legacy")
-        _merge_pdfs_legacy(pdf1_path, pdf2_path, output_path)
-
-
-def _merge_pdfs_ng(pdf1_path, pdf2_path, output_path):
     import PyPDF2  # PyPDF2这个库有严重的内存泄露问题，把它放到子进程中运行，从而方便内存的释放
-    from PyPDF2.generic import NameObject, TextStringObject, ArrayObject, FloatObject, NumberObject
+    from PyPDF2.generic import NameObject, TextStringObject,ArrayObject,FloatObject,NumberObject
 
     Percent = 1
     # raise RuntimeError('PyPDF2 has a serious memory leak problem, please use other tools to merge PDF files.')
@@ -697,206 +688,65 @@ def _merge_pdfs_ng(pdf1_path, pdf2_path, output_path):
                     ),
                     0,
                 )
-                if "/Annots" in new_page:
+                if '/Annots' in page1:
-                    annotations = new_page["/Annots"]
+                    page1_annot_id = [annot.idnum for annot in page1['/Annots']]
+                else:
+                    page1_annot_id = []
+
+                if '/Annots' in page2:
+                    page2_annot_id = [annot.idnum for annot in page2['/Annots']]
+                else:
+                    page2_annot_id = []
+                if '/Annots' in new_page:
+                    annotations = new_page['/Annots']
                     for i, annot in enumerate(annotations):
                         annot_obj = annot.get_object()
 
                         # 检查注释类型是否是链接（/Link）
-                        if annot_obj.get("/Subtype") == "/Link":
+                        if annot_obj.get('/Subtype') == '/Link':
                             # 检查是否为内部链接跳转（/GoTo）或外部URI链接（/URI）
-                            action = annot_obj.get("/A")
+                            action = annot_obj.get('/A')
                             if action:
 
-                                if "/S" in action and action["/S"] == "/GoTo":
+                                if '/S' in action and action['/S'] == '/GoTo':
                                     # 内部链接：跳转到文档中的某个页面
-                                    dest = action.get("/D")  # 目标页或目标位置
+                                    dest = action.get('/D')  # 目标页或目标位置
-                                    # if dest and annot.idnum in page2_annot_id:
+                                    if dest and annot.idnum in page2_annot_id:
-                                    # if dest in pdf2_reader.named_destinations:
+                                        # 获取原始文件中跳转信息，包括跳转页面
-                                    if dest and page2.annotations:
+                                        destination = pdf2_reader.named_destinations[dest]
-                                        if annot in page2.annotations:
+                                        page_number = pdf2_reader.get_destination_page_number(destination)
-                                            # 获取原始文件中跳转信息，包括跳转页面
+                                        #更新跳转信息，跳转到对应的页面和，指定坐标 (100, 150)，缩放比例为 100%
-                                            destination = pdf2_reader.named_destinations[
+                                        #“/D”:[10,'/XYZ',100,100,0]
-                                                dest
+                                        annot_obj['/A'].update({
-                                            ]
+                                            NameObject("/D"): ArrayObject([NumberObject(page_number),destination.dest_array[1], FloatObject(destination.dest_array[2] + int(page1.mediaBox.getWidth())) ,destination.dest_array[3],destination.dest_array[4]])  # 确保键和值是 PdfObject
-                                            page_number = (
+                                        })
-                                                pdf2_reader.get_destination_page_number(
+                                        rect = annot_obj.get('/Rect')
-                                                    destination
+                                        # 更新点击坐标
-                                                )
+                                        rect = ArrayObject([FloatObject(rect[0]+ int(page1.mediaBox.getWidth())),rect[1],
-                                            )
+                                                            FloatObject(rect[2]+int(page1.mediaBox.getWidth())),rect[3] ])
-                                            # 更新跳转信息，跳转到对应的页面和，指定坐标 (100, 150)，缩放比例为 100%
+                                        annot_obj.update({
-                                            # “/D”:[10,'/XYZ',100,100,0]
+                                            NameObject("/Rect"): rect  # 确保键和值是 PdfObject
-                                            if destination.dest_array[1] == "/XYZ":
+                                        })
-                                                annot_obj["/A"].update(
+                                    if dest and annot.idnum in page1_annot_id:
-                                                    {
+                                        # 获取原始文件中跳转信息，包括跳转页面
-                                                        NameObject("/D"): ArrayObject(
+                                        destination = pdf1_reader.named_destinations[dest]
-                                                            [
+                                        page_number = pdf1_reader.get_destination_page_number(destination)
-                                                                NumberObject(page_number),
+                                        #更新跳转信息，跳转到对应的页面和，指定坐标 (100, 150)，缩放比例为 100%
-                                                                destination.dest_array[1],
+                                        #“/D”:[10,'/XYZ',100,100,0]
-                                                                FloatObject(
+                                        annot_obj['/A'].update({
-                                                                    destination.dest_array[
+                                            NameObject("/D"): ArrayObject([NumberObject(page_number),destination.dest_array[1], FloatObject(destination.dest_array[2]) ,destination.dest_array[3],destination.dest_array[4]])  # 确保键和值是 PdfObject
-                                                                        2
+                                        })
-                                                                    ]
+                                        rect = annot_obj.get('/Rect')
-                                                                    + int(
+                                        rect = ArrayObject([FloatObject(rect[0]),rect[1],
-                                                                        page1.mediaBox.getWidth()
+                                                            FloatObject(rect[2]),rect[3] ])
-                                                                    )
+                                        annot_obj.update({
-                                                                ),
+                                            NameObject("/Rect"): rect  # 确保键和值是 PdfObject
-                                                                destination.dest_array[3],
+                                        })
-                                                                destination.dest_array[4],
-                                                            ]
-                                                        )  # 确保键和值是 PdfObject
-                                                    }
-                                                )
-                                            else:
-                                                annot_obj["/A"].update(
-                                                    {
-                                                        NameObject("/D"): ArrayObject(
-                                                            [
-                                                                NumberObject(page_number),
-                                                                destination.dest_array[1],
-                                                            ]
-                                                        )  # 确保键和值是 PdfObject
-                                                    }
-                                                )
 
-                                            rect = annot_obj.get("/Rect")
+                                elif '/S' in action and action['/S'] == '/URI':
-                                            # 更新点击坐标
-                                            rect = ArrayObject(
-                                                [
-                                                    FloatObject(
-                                                        rect[0]
-                                                        + int(page1.mediaBox.getWidth())
-                                                    ),
-                                                    rect[1],
-                                                    FloatObject(
-                                                        rect[2]
-                                                        + int(page1.mediaBox.getWidth())
-                                                    ),
-                                                    rect[3],
-                                                ]
-                                            )
-                                            annot_obj.update(
-                                                {
-                                                    NameObject(
-                                                        "/Rect"
-                                                    ): rect  # 确保键和值是 PdfObject
-                                                }
-                                            )
-                                    # if dest and annot.idnum in page1_annot_id:
-                                    # if dest in pdf1_reader.named_destinations:
-                                    if dest and page1.annotations:
-                                        if annot in page1.annotations:
-                                            # 获取原始文件中跳转信息，包括跳转页面
-                                            destination = pdf1_reader.named_destinations[
-                                                dest
-                                            ]
-                                            page_number = (
-                                                pdf1_reader.get_destination_page_number(
-                                                    destination
-                                                )
-                                            )
-                                            # 更新跳转信息，跳转到对应的页面和，指定坐标 (100, 150)，缩放比例为 100%
-                                            # “/D”:[10,'/XYZ',100,100,0]
-                                            if destination.dest_array[1] == "/XYZ":
-                                                annot_obj["/A"].update(
-                                                    {
-                                                        NameObject("/D"): ArrayObject(
-                                                            [
-                                                                NumberObject(page_number),
-                                                                destination.dest_array[1],
-                                                                FloatObject(
-                                                                    destination.dest_array[
-                                                                        2
-                                                                    ]
-                                                                ),
-                                                                destination.dest_array[3],
-                                                                destination.dest_array[4],
-                                                            ]
-                                                        )  # 确保键和值是 PdfObject
-                                                    }
-                                                )
-                                            else:
-                                                annot_obj["/A"].update(
-                                                    {
-                                                        NameObject("/D"): ArrayObject(
-                                                            [
-                                                                NumberObject(page_number),
-                                                                destination.dest_array[1],
-                                                            ]
-                                                        )  # 确保键和值是 PdfObject
-                                                    }
-                                                )
-
-                                            rect = annot_obj.get("/Rect")
-                                            rect = ArrayObject(
-                                                [
-                                                    FloatObject(rect[0]),
-                                                    rect[1],
-                                                    FloatObject(rect[2]),
-                                                    rect[3],
-                                                ]
-                                            )
-                                            annot_obj.update(
-                                                {
-                                                    NameObject(
-                                                        "/Rect"
-                                                    ): rect  # 确保键和值是 PdfObject
-                                                }
-                                            )
-
-                                elif "/S" in action and action["/S"] == "/URI":
                                     # 外部链接：跳转到某个URI
-                                    uri = action.get("/URI")
+                                    uri = action.get('/URI')
                 output_writer.addPage(new_page)
-            # Save the merged PDF file
-            with open(output_path, "wb") as output_file:
-                output_writer.write(output_file)
-
-
-def _merge_pdfs_legacy(pdf1_path, pdf2_path, output_path):
-    import PyPDF2  # PyPDF2这个库有严重的内存泄露问题，把它放到子进程中运行，从而方便内存的释放
-
-    Percent = 0.95
-    # raise RuntimeError('PyPDF2 has a serious memory leak problem, please use other tools to merge PDF files.')
-    # Open the first PDF file
-    with open(pdf1_path, "rb") as pdf1_file:
-        pdf1_reader = PyPDF2.PdfFileReader(pdf1_file)
-        # Open the second PDF file
-        with open(pdf2_path, "rb") as pdf2_file:
-            pdf2_reader = PyPDF2.PdfFileReader(pdf2_file)
-            # Create a new PDF file to store the merged pages
-            output_writer = PyPDF2.PdfFileWriter()
-            # Determine the number of pages in each PDF file
-            num_pages = max(pdf1_reader.numPages, pdf2_reader.numPages)
-            # Merge the pages from the two PDF files
-            for page_num in range(num_pages):
-                # Add the page from the first PDF file
-                if page_num < pdf1_reader.numPages:
-                    page1 = pdf1_reader.getPage(page_num)
-                else:
-                    page1 = PyPDF2.PageObject.createBlankPage(pdf1_reader)
-                # Add the page from the second PDF file
-                if page_num < pdf2_reader.numPages:
-                    page2 = pdf2_reader.getPage(page_num)
-                else:
-                    page2 = PyPDF2.PageObject.createBlankPage(pdf1_reader)
-                # Create a new empty page with double width
-                new_page = PyPDF2.PageObject.createBlankPage(
-                    width=int(
-                        int(page1.mediaBox.getWidth())
-                        + int(page2.mediaBox.getWidth()) * Percent
-                    ),
-                    height=max(page1.mediaBox.getHeight(), page2.mediaBox.getHeight()),
-                )
-                new_page.mergeTranslatedPage(page1, 0, 0)
-                new_page.mergeTranslatedPage(
-                    page2,
-                    int(
-                        int(page1.mediaBox.getWidth())
-                        - int(page2.mediaBox.getWidth()) * (1 - Percent)
-                    ),
-                    0,
-                )
                 output_writer.addPage(new_page)
             # Save the merged PDF file
             with open(output_path, "wb") as output_file:

crazy_functions/rag_fns/llama_index_worker.py

@@ -1,13 +1,17 @@
+import llama_index
+import os
 import atexit
 from loguru import logger
 from typing import List
-
 from llama_index.core import Document
-from llama_index.core.ingestion import run_transformations
 from llama_index.core.schema import TextNode
-
-from crazy_functions.rag_fns.vector_store_index import GptacVectorStoreIndex
 from request_llms.embed_models.openai_embed import OpenAiEmbeddingModel
+from shared_utils.connect_void_terminal import get_chat_default_kwargs
+from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
+from crazy_functions.rag_fns.vector_store_index import GptacVectorStoreIndex
+from llama_index.core.ingestion import run_transformations
+from llama_index.core import PromptTemplate
+from llama_index.core.response_synthesizers import TreeSummarize
 
 DEFAULT_QUERY_GENERATION_PROMPT = """\
 Now, you have context information as below:
@@ -59,7 +63,7 @@ class SaveLoad():
     def purge(self):
         import shutil
         shutil.rmtree(self.checkpoint_dir, ignore_errors=True)
-        self.vs_index = self.create_new_vs(self.checkpoint_dir)
+        self.vs_index = self.create_new_vs()
 
 
 class LlamaIndexRagWorker(SaveLoad):
@@ -71,7 +75,7 @@ class LlamaIndexRagWorker(SaveLoad):
         if auto_load_checkpoint:
             self.vs_index = self.load_from_checkpoint(checkpoint_dir)
         else:
-            self.vs_index = self.create_new_vs()
+            self.vs_index = self.create_new_vs(checkpoint_dir)
         atexit.register(lambda: self.save_to_checkpoint(checkpoint_dir))
 
     def assign_embedding_model(self):
@@ -87,52 +91,40 @@ class LlamaIndexRagWorker(SaveLoad):
         logger.info('oo --------inspect_vector_store end--------')
         return vector_store_preview
 
-    def add_documents_to_vector_store(self, document_list: List[Document]):
+    def add_documents_to_vector_store(self, document_list):
-        """
+        documents = [Document(text=t) for t in document_list]
-        Adds a list of Document objects to the vector store after processing.
-        """
-        documents = document_list
         documents_nodes = run_transformations(
-            documents,  # type: ignore
+                        documents,  # type: ignore
-            self.vs_index._transformations,
+                        self.vs_index._transformations,
-            show_progress=True
+                        show_progress=True
-        )
+                    )
         self.vs_index.insert_nodes(documents_nodes)
-        if self.debug_mode:
+        if self.debug_mode: self.inspect_vector_store()
-            self.inspect_vector_store()
 
-    def add_text_to_vector_store(self, text: str):
+    def add_text_to_vector_store(self, text):
         node = TextNode(text=text)
         documents_nodes = run_transformations(
-            [node],
+                        [node],
-            self.vs_index._transformations,
+                        self.vs_index._transformations,
-            show_progress=True
+                        show_progress=True
-        )
+                    )
         self.vs_index.insert_nodes(documents_nodes)
-        if self.debug_mode:
+        if self.debug_mode: self.inspect_vector_store()
-            self.inspect_vector_store()
 
     def remember_qa(self, question, answer):
         formatted_str = QUESTION_ANSWER_RECORD.format(question=question, answer=answer)
         self.add_text_to_vector_store(formatted_str)
 
     def retrieve_from_store_with_query(self, query):
-        if self.debug_mode:
+        if self.debug_mode: self.inspect_vector_store()
-            self.inspect_vector_store()
         retriever = self.vs_index.as_retriever()
         return retriever.retrieve(query)
 
     def build_prompt(self, query, nodes):
         context_str = self.generate_node_array_preview(nodes)
         return DEFAULT_QUERY_GENERATION_PROMPT.format(context_str=context_str, query_str=query)
-
+        
     def generate_node_array_preview(self, nodes):
         buf = "\n".join(([f"(No.{i+1} | score {n.score:.3f}): {n.text}" for i, n in enumerate(nodes)]))
         if self.debug_mode: logger.info(buf)
         return buf
-
-    def purge_vector_store(self):
-        """
-        Purges the current vector store and creates a new one.
-        """
-        self.purge()

crazy_functions/rag_fns/rag_file_support.py

@@ -1,22 +0,0 @@
-import os
-from llama_index.core import SimpleDirectoryReader
-
-supports_format = ['.csv', '.docx', '.epub', '.ipynb',  '.mbox', '.md', '.pdf',  '.txt', '.ppt',
-                   '.pptm', '.pptx']
-
-
-# 修改后的 extract_text 函数，结合 SimpleDirectoryReader 和自定义解析逻辑
-def extract_text(file_path):
-    _, ext = os.path.splitext(file_path.lower())
-
-    # 使用 SimpleDirectoryReader 处理它支持的文件格式
-    if ext in supports_format:
-        try:
-            reader = SimpleDirectoryReader(input_files=[file_path])
-            documents = reader.load_data()
-            if len(documents) > 0:
-                return documents[0].text
-        except Exception as e:
-            pass
-
-    return None

docs/Dockerfile+JittorLLM

@@ -0,0 +1 @@
+# 此Dockerfile不再维护，请前往docs/GithubAction+JittorLLMs

docs/GithubAction+AllCapacityBeta

@@ -0,0 +1,57 @@
+# docker build -t gpt-academic-all-capacity -f docs/GithubAction+AllCapacity  --network=host --build-arg http_proxy=http://localhost:10881 --build-arg https_proxy=http://localhost:10881 .
+# docker build -t gpt-academic-all-capacity -f docs/GithubAction+AllCapacityBeta  --network=host .
+# docker run -it --net=host gpt-academic-all-capacity  bash
+
+# 从NVIDIA源，从而支持显卡（检查宿主的nvidia-smi中的cuda版本必须>=11.3）
+FROM fuqingxu/11.3.1-runtime-ubuntu20.04-with-texlive:latest
+
+# edge-tts需要的依赖，某些pip包所需的依赖
+RUN apt update && apt install ffmpeg build-essential -y
+
+# use python3 as the system default python
+WORKDIR /gpt
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.8
+
+# # 非必要步骤，更换pip源 （以下三行，可以删除）
+# RUN echo '[global]' > /etc/pip.conf && \
+#     echo 'index-url = https://mirrors.aliyun.com/pypi/simple/' >> /etc/pip.conf && \
+#     echo 'trusted-host = mirrors.aliyun.com' >> /etc/pip.conf
+
+# 下载pytorch
+RUN python3 -m pip install torch torchvision --extra-index-url https://download.pytorch.org/whl/cu113
+# 准备pip依赖
+RUN python3 -m pip install openai numpy arxiv rich
+RUN python3 -m pip install colorama Markdown pygments pymupdf
+RUN python3 -m pip install python-docx moviepy pdfminer
+RUN python3 -m pip install zh_langchain==0.2.1 pypinyin
+RUN python3 -m pip install rarfile py7zr
+RUN python3 -m pip install aliyun-python-sdk-core==2.13.3 pyOpenSSL webrtcvad scipy git+https://github.com/aliyun/alibabacloud-nls-python-sdk.git
+# 下载分支
+WORKDIR /gpt
+RUN git clone --depth=1 https://github.com/binary-husky/gpt_academic.git
+WORKDIR /gpt/gpt_academic
+RUN git clone --depth=1 https://github.com/OpenLMLab/MOSS.git request_llms/moss
+
+RUN python3 -m pip install -r requirements.txt
+RUN python3 -m pip install -r request_llms/requirements_moss.txt
+RUN python3 -m pip install -r request_llms/requirements_qwen.txt
+RUN python3 -m pip install -r request_llms/requirements_chatglm.txt
+RUN python3 -m pip install -r request_llms/requirements_newbing.txt
+RUN python3 -m pip install nougat-ocr
+
+
+# 预热Tiktoken模块
+RUN python3  -c 'from check_proxy import warm_up_modules; warm_up_modules()'
+
+# 安装知识库插件的额外依赖
+RUN apt-get update && apt-get install libgl1 -y
+RUN pip3 install transformers protobuf langchain sentence-transformers  faiss-cpu nltk beautifulsoup4 bitsandbytes tabulate icetk --upgrade
+RUN pip3 install unstructured[all-docs] --upgrade
+RUN python3  -c 'from check_proxy import warm_up_vectordb; warm_up_vectordb()'
+RUN rm -rf /usr/local/lib/python3.8/dist-packages/tests
+
+
+# COPY .cache /root/.cache
+# COPY config_private.py config_private.py
+# 启动
+CMD ["python3", "-u", "main.py"]

docs/GithubAction+NoLocal+Latex+Arm

@@ -1,25 +0,0 @@
-# 此Dockerfile适用于“无本地模型”的环境构建，如果需要使用chatglm等本地模型，请参考 docs/Dockerfile+ChatGLM
-# - 1 修改 `config.py`
-# - 2 构建 docker build -t gpt-academic-nolocal-latex -f docs/GithubAction+NoLocal+Latex .
-# - 3 运行 docker run -v /home/fuqingxu/arxiv_cache:/root/arxiv_cache --rm -it --net=host gpt-academic-nolocal-latex
-
-FROM menghuan1918/ubuntu_uv_ctex:latest
-ENV DEBIAN_FRONTEND=noninteractive
-SHELL ["/bin/bash", "-c"]
-WORKDIR /gpt
-COPY . .
-RUN /root/.cargo/bin/uv venv --seed \
-    && source .venv/bin/activate \
-    && /root/.cargo/bin/uv pip install openai numpy arxiv rich colorama Markdown pygments pymupdf python-docx pdfminer \
-    && /root/.cargo/bin/uv pip install -r requirements.txt \
-    && /root/.cargo/bin/uv clean
-
-# 对齐python3
-RUN rm -f /usr/bin/python3 && ln -s /gpt/.venv/bin/python /usr/bin/python3
-RUN rm -f /usr/bin/python && ln -s /gpt/.venv/bin/python /usr/bin/python
-
-# 可选步骤，用于预热模块
-RUN python3 -c 'from check_proxy import warm_up_modules; warm_up_modules()'
-
-# 启动
-CMD ["python3", "-u", "main.py"]

request_llms/bridge_all.py

@@ -256,8 +256,6 @@ model_info = {
         "max_token": 128000,
         "tokenizer": tokenizer_gpt4,
         "token_cnt": get_token_num_gpt4,
-        "openai_disable_system_prompt": True,
-        "openai_disable_stream": True,
     },
     "o1-mini": {
         "fn_with_ui": chatgpt_ui,
@@ -266,8 +264,6 @@ model_info = {
         "max_token": 128000,
         "tokenizer": tokenizer_gpt4,
         "token_cnt": get_token_num_gpt4,
-        "openai_disable_system_prompt": True,
-        "openai_disable_stream": True,
     },
 
     "gpt-4-turbo": {
@@ -1285,3 +1281,4 @@ def predict(inputs:str, llm_kwargs:dict, plugin_kwargs:dict, chatbot,
 
     # 更新一下llm_kwargs的参数，否则会出现参数不匹配的问题
     yield from method(inputs, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, stream, additional_fn)
+

request_llms/bridge_chatgpt.py

@@ -202,13 +202,10 @@ def predict_no_ui_long_connection(inputs:str, llm_kwargs:dict, history:list=[],
                     if (time.time()-observe_window[1]) > watch_dog_patience:
                         raise RuntimeError("用户取消了程序。")
         else: raise RuntimeError("意外Json结构："+delta)
-
+    if json_data and json_data['finish_reason'] == 'content_filter':
-    finish_reason = json_data.get('finish_reason', None) if json_data else None
+        raise RuntimeError("由于提问含不合规内容被Azure过滤。")
-    if finish_reason == 'content_filter':
+    if json_data and json_data['finish_reason'] == 'length':
-        raise RuntimeError("由于提问含不合规内容被过滤。")
-    if finish_reason == 'length':
         raise ConnectionAbortedError("正常结束，但显示Token不足，导致输出不完整，请削减单次输入的文本量。")
-
     return result
 
 
@@ -539,3 +536,4 @@ def generate_payload(inputs:str, llm_kwargs:dict, history:list, system_prompt:st
 
     return headers,payload
 
+

requirements.txt

@@ -2,15 +2,14 @@ https://public.agent-matrix.com/publish/gradio-3.32.10-py3-none-any.whl
 fastapi==0.110
 gradio-client==0.8
 pypdf2==2.12.1
-httpx<=0.25.2
 zhipuai==2.0.1
 tiktoken>=0.3.3
 requests[socks]
-pydantic==2.9.2
+pydantic==2.5.2
+llama-index~=0.10
 protobuf==3.20
 transformers>=4.27.1,<4.42
 scipdf_parser>=0.52
-spacy==3.7.4
 anthropic>=0.18.1
 python-markdown-math
 pymdown-extensions
@@ -33,14 +32,3 @@ loguru
 arxiv
 numpy
 rich
-
-
-llama-index-core==0.10.68
-llama-index-legacy==0.9.48
-llama-index-readers-file==0.1.33
-llama-index-readers-llama-parse==0.1.6
-llama-index-embeddings-azure-openai==0.1.10
-llama-index-embeddings-openai==0.1.10
-llama-parse==0.4.9
-mdit-py-plugins>=0.3.3
-linkify-it-py==2.0.3

tests/test_anim_gen.py

@@ -1,12 +0,0 @@
-"""
-对项目中的各个插件进行测试。运行方法：直接运行 python tests/test_plugins.py
-"""
-
-import init_test
-import os, sys
-
-
-if __name__ == "__main__":
-    from test_utils import plugin_test
-
-    plugin_test(plugin='crazy_functions.数学动画生成manim->动画生成', main_input="A point moving along function culve y=sin(x), starting from x=0 and stop at x=4*\pi.")

tests/test_latex_auto_correct.py

@@ -19,8 +19,4 @@ if __name__ == "__main__":
     plugin_test = importlib.import_module('test_utils').plugin_test
 
 
-    # plugin_test(plugin='crazy_functions.Latex_Function->Latex翻译中文并重新编译PDF', main_input="2203.01927")
+    plugin_test(plugin='crazy_functions.Latex_Function->Latex翻译中文并重新编译PDF', main_input="2203.01927")
-    # plugin_test(plugin='crazy_functions.Latex_Function->Latex翻译中文并重新编译PDF', main_input="gpt_log/arxiv_cache/2203.01927/workfolder")
-    # plugin_test(plugin='crazy_functions.Latex_Function->Latex翻译中文并重新编译PDF', main_input="2410.05779")
-    plugin_test(plugin='crazy_functions.Latex_Function->Latex翻译中文并重新编译PDF', main_input="gpt_log/default_user/workfolder")
-

version

@@ -1,5 +1,5 @@
 {
-  "version": 3.90,
+  "version": 3.83,
   "show_feature": true,
-  "new_feature": "增加RAG组件 <-> 升级多合一主提交键"
+  "new_feature": "增加欢迎页面 <-> 优化图像生成插件 <-> 添加紫东太初大模型支持 <-> 保留主题选择 <-> 支持更复杂的插件框架 <-> 上传文件时显示进度条"
 }