add legacy fallback option

.github/workflows/build-with-latex-arm.yml

@@ -1,51 +0,0 @@
-# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images#publishing-images-to-github-packages
-name: build-with-latex-arm
-
-on:
-  push:
-    branches:
-      - "master"
-
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}_with_latex_arm
-
-jobs:
-  build-and-push-image:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-
-    steps:
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-
-      - name: Build and push Docker image
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          push: true
-          platforms: linux/arm64
-          file: docs/GithubAction+NoLocal+Latex+Arm
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}

crazy_functions/Latex_Function.py

@@ -3,7 +3,7 @@ from toolbox import CatchException, report_exception, update_ui_lastest_msg, zip
 from functools import partial
 from loguru import logger
 
-import glob, os, requests, time, json, tarfile, threading
+import glob, os, requests, time, json, tarfile
 
 pj = os.path.join
 ARXIV_CACHE_DIR = get_conf("ARXIV_CACHE_DIR")
@@ -138,43 +138,25 @@ def arxiv_download(chatbot, history, txt, allow_cache=True):
     cached_translation_pdf = check_cached_translation_pdf(arxiv_id)
     if cached_translation_pdf and allow_cache: return cached_translation_pdf, arxiv_id
 
-    extract_dst = pj(ARXIV_CACHE_DIR, arxiv_id, 'extract')
+    url_tar = url_.replace('/abs/', '/e-print/')
     translation_dir = pj(ARXIV_CACHE_DIR, arxiv_id, 'e-print')
-    dst = pj(translation_dir, arxiv_id + '.tar')
+    extract_dst = pj(ARXIV_CACHE_DIR, arxiv_id, 'extract')
     os.makedirs(translation_dir, exist_ok=True)
-    # <-------------- download arxiv source file ------------->
 
-    def fix_url_and_download():
+    # <-------------- download arxiv source file ------------->
-        # for url_tar in [url_.replace('/abs/', '/e-print/'), url_.replace('/abs/', '/src/')]:
+    dst = pj(translation_dir, arxiv_id + '.tar')
-        for url_tar in [url_.replace('/abs/', '/src/'), url_.replace('/abs/', '/e-print/')]:
+    if os.path.exists(dst):
+        yield from update_ui_lastest_msg("调用缓存", chatbot=chatbot, history=history)  # 刷新界面
+    else:
+        yield from update_ui_lastest_msg("开始下载", chatbot=chatbot, history=history)  # 刷新界面
         proxies = get_conf('proxies')
         r = requests.get(url_tar, proxies=proxies)
-            if r.status_code == 200:
         with open(dst, 'wb+') as f:
             f.write(r.content)
-                return True
-        return False
-
-    if os.path.exists(dst) and allow_cache:
-        yield from update_ui_lastest_msg(f"调用缓存 {arxiv_id}", chatbot=chatbot, history=history)  # 刷新界面
-        success = True
-    else:
-        yield from update_ui_lastest_msg(f"开始下载 {arxiv_id}", chatbot=chatbot, history=history)  # 刷新界面
-        success = fix_url_and_download()
-        yield from update_ui_lastest_msg(f"下载完成 {arxiv_id}", chatbot=chatbot, history=history)  # 刷新界面
-
-
-    if not success:
-        yield from update_ui_lastest_msg(f"下载失败 {arxiv_id}", chatbot=chatbot, history=history)
-        raise tarfile.ReadError(f"论文下载失败 {arxiv_id}")
-
     # <-------------- extract file ------------->
+    yield from update_ui_lastest_msg("下载完成", chatbot=chatbot, history=history)  # 刷新界面
     from toolbox import extract_archive
-    try:
     extract_archive(file_path=dst, dest_dir=extract_dst)
-    except tarfile.ReadError:
-        os.remove(dst)
-        raise tarfile.ReadError(f"论文下载失败")
     return extract_dst, arxiv_id
 
 
@@ -338,17 +320,11 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot,
     # <-------------- more requirements ------------->
     if ("advanced_arg" in plugin_kwargs) and (plugin_kwargs["advanced_arg"] == ""): plugin_kwargs.pop("advanced_arg")
     more_req = plugin_kwargs.get("advanced_arg", "")
-
+    no_cache = more_req.startswith("--no-cache")
-    no_cache = ("--no-cache" in more_req)
+    if no_cache: more_req.lstrip("--no-cache")
-    if no_cache: more_req = more_req.replace("--no-cache", "").strip()
-
-    allow_gptac_cloud_io = ("--allow-cloudio" in more_req)  # 从云端下载翻译结果，以及上传翻译结果到云端
-    if allow_gptac_cloud_io: more_req = more_req.replace("--allow-cloudio", "").strip()
-
     allow_cache = not no_cache
     _switch_prompt_ = partial(switch_prompt, more_requirement=more_req)
 
-
     # <-------------- check deps ------------->
     try:
         import glob, os, time, subprocess
@@ -375,20 +351,6 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot,
         yield from update_ui(chatbot=chatbot, history=history)  # 刷新界面
         return
 
-    # #################################################################
-    if allow_gptac_cloud_io and arxiv_id:
-        # 访问 GPTAC学术云，查询云端是否存在该论文的翻译版本
-        from crazy_functions.latex_fns.latex_actions import check_gptac_cloud
-        success, downloaded = check_gptac_cloud(arxiv_id, chatbot)
-        if success:
-            chatbot.append([
-                f"检测到GPTAC云端存在翻译版本, 如果不满意翻译结果, 请禁用云端分享, 然后重新执行。", 
-                None
-            ])
-            yield from update_ui(chatbot=chatbot, history=history)
-            return
-    #################################################################
-
     if os.path.exists(txt):
         project_folder = txt
     else:
@@ -426,21 +388,14 @@ def Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot,
     # <-------------- zip PDF ------------->
     zip_res = zip_result(project_folder)
     if success:
-        if allow_gptac_cloud_io and arxiv_id:
-            # 如果用户允许，我们将翻译好的arxiv论文PDF上传到GPTAC学术云
-            from crazy_functions.latex_fns.latex_actions import upload_to_gptac_cloud_if_user_allow
-            threading.Thread(target=upload_to_gptac_cloud_if_user_allow, 
-                args=(chatbot, arxiv_id), daemon=True).start()
-
         chatbot.append((f"成功啦", '请查收结果（压缩包）...'))
-        yield from update_ui(chatbot=chatbot, history=history)
+        yield from update_ui(chatbot=chatbot, history=history);
         time.sleep(1)  # 刷新界面
         promote_file_to_downloadzone(file=zip_res, chatbot=chatbot)
-
     else:
         chatbot.append((f"失败了",
                         '虽然PDF生成失败了, 但请查收结果（压缩包）, 内含已经翻译的Tex文档, 您可以到Github Issue区, 用该压缩包进行反馈。如系统是Linux，请检查系统字体（见Github wiki） ...'))
-        yield from update_ui(chatbot=chatbot, history=history)
+        yield from update_ui(chatbot=chatbot, history=history);
         time.sleep(1)  # 刷新界面
         promote_file_to_downloadzone(file=zip_res, chatbot=chatbot)
 

crazy_functions/Latex_Function_Wrap.py

@@ -30,9 +30,6 @@ class Arxiv_Localize(GptAcademicPluginTemplate):
                             default_value="", type="string").model_dump_json(), # 高级参数输入区，自动同步
             "allow_cache":
                 ArgProperty(title="是否允许从缓存中调取结果", options=["允许缓存", "从头执行"], default_value="允许缓存", description="无", type="dropdown").model_dump_json(),
-            "allow_cloudio":
-                ArgProperty(title="是否允许向GPTAC学术云共享翻译结果", options=["允许", "禁止"], default_value="禁止", description="人人为我，我为人人", type="dropdown").model_dump_json(),
-
         }
         return gui_definition
 
@@ -41,14 +38,9 @@ class Arxiv_Localize(GptAcademicPluginTemplate):
         执行插件
         """
         allow_cache = plugin_kwargs["allow_cache"]
-        allow_cloudio = plugin_kwargs["allow_cloudio"]
         advanced_arg = plugin_kwargs["advanced_arg"]
 
         if allow_cache == "从头执行": plugin_kwargs["advanced_arg"] = "--no-cache " + plugin_kwargs["advanced_arg"]
-
-        # 从云端下载翻译结果，以及上传翻译结果到云端；人人为我，我为人人。
-        if allow_cloudio == "允许": plugin_kwargs["advanced_arg"] = "--allow-cloudio " + plugin_kwargs["advanced_arg"]
-
         yield from Latex翻译中文并重新编译PDF(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request)
 
 

crazy_functions/Rag_Interface.py

@@ -1,11 +1,4 @@
-import os,glob
-from typing import List
-
-from shared_utils.fastapi_server import validate_path_safety
-
-from toolbox import report_exception
 from toolbox import CatchException, update_ui, get_conf, get_log_folder, update_ui_lastest_msg
-from shared_utils.fastapi_server import validate_path_safety
 from crazy_functions.crazy_utils import input_clipping
 from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
 
@@ -14,37 +7,6 @@ MAX_HISTORY_ROUND = 5
 MAX_CONTEXT_TOKEN_LIMIT = 4096
 REMEMBER_PREVIEW = 1000
 
-@CatchException
-def handle_document_upload(files: List[str], llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request, rag_worker):
-    """
-    Handles document uploads by extracting text and adding it to the vector store.
-    """
-    from llama_index.core import Document
-    from crazy_functions.rag_fns.rag_file_support import extract_text, supports_format
-    user_name = chatbot.get_user()
-    checkpoint_dir = get_log_folder(user_name, plugin_name='experimental_rag')
-
-    for file_path in files:
-        try:
-            validate_path_safety(file_path, user_name)
-            text = extract_text(file_path)
-            if text is None:
-                chatbot.append(
-                    [f"上传文件: {os.path.basename(file_path)}", f"文件解析失败，无法提取文本内容，请更换文件。失败原因可能为：1.文档格式过于复杂；2. 不支持的文件格式，支持的文件格式后缀有:" + ", ".join(supports_format)])
-            else:
-                chatbot.append(
-                    [f"上传文件: {os.path.basename(file_path)}", f"上传文件前50个字符为:{text[:50]}。"])
-                document = Document(text=text, metadata={"source": file_path})
-                rag_worker.add_documents_to_vector_store([document])
-                chatbot.append([f"上传文件: {os.path.basename(file_path)}", "文件已成功添加到知识库。"])
-        except Exception as e:
-            report_exception(chatbot, history, a=f"处理文件: {file_path}", b=str(e))
-
-    yield from update_ui(chatbot=chatbot, history=history)  # 刷新界面
-
-
-
-# Main Q&A function with document upload support
 @CatchException
 def Rag问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request):
 
@@ -61,7 +23,6 @@ def Rag问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, u
     # 1. we retrieve rag worker from global context
     user_name = chatbot.get_user()
     checkpoint_dir = get_log_folder(user_name, plugin_name='experimental_rag')
-
     if user_name in RAG_WORKER_REGISTER:
         rag_worker = RAG_WORKER_REGISTER[user_name]
     else:
@@ -69,40 +30,21 @@ def Rag问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, u
             user_name, 
             llm_kwargs, 
             checkpoint_dir=checkpoint_dir, 
-            auto_load_checkpoint=True
+            auto_load_checkpoint=True)
-        )
-
     current_context = f"{VECTOR_STORE_TYPE} @ {checkpoint_dir}"
     tip = "提示：输入“清空向量数据库”可以清空RAG向量数据库"
-
+    if txt == "清空向量数据库":
-    # 2. Handle special commands
-    if os.path.exists(txt) and os.path.isdir(txt):
-        project_folder = txt
-        validate_path_safety(project_folder, chatbot.get_user())
-        # Extract file paths from the user input
-        # Assuming the user inputs file paths separated by commas after the command
-        file_paths =  [f for f in glob.glob(f'{project_folder}/**/*', recursive=True)]
-        chatbot.append([txt, f'正在处理上传的文档 ({current_context}) ...'])
-        yield from update_ui(chatbot=chatbot, history=history)  # 刷新界面
-
-        yield from handle_document_upload(file_paths, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, user_request, rag_worker)
-        return
-
-    elif txt == "清空向量数据库":
         chatbot.append([txt, f'正在清空 ({current_context}) ...'])
         yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
-        rag_worker.purge_vector_store()
+        rag_worker.purge()
         yield from update_ui_lastest_msg('已清空', chatbot, history, delay=0) # 刷新界面
         return
 
-    else:
-        report_exception(chatbot, history, a=f"上传文件路径错误: {txt}", b="请检查并提供正确路径。")
-
-    # 3. Normal Q&A processing
     chatbot.append([txt, f'正在召回知识 ({current_context}) ...'])
     yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
 
-    # 4. Clip history to reduce token consumption
+    # 2. clip history to reduce token consumption
+    #   2-1. reduce chat round
     txt_origin = txt
 
     if len(history) > MAX_HISTORY_ROUND * 2:
@@ -110,47 +52,41 @@ def Rag问答(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, u
     txt_clip, history, flags = input_clipping(txt, history, max_token_limit=MAX_CONTEXT_TOKEN_LIMIT, return_clip_flags=True)
     input_is_clipped_flag = (flags["original_input_len"] != flags["clipped_input_len"])
 
-    # 5. If input is clipped, add input to vector store before retrieve
+    #   2-2. if input is clipped, add input to vector store before retrieve
     if input_is_clipped_flag:
         yield from update_ui_lastest_msg('检测到长输入, 正在向量化 ...', chatbot, history, delay=0) # 刷新界面
-        # Save input to vector store
+        # save input to vector store
         rag_worker.add_text_to_vector_store(txt_origin)
         yield from update_ui_lastest_msg('向量化完成 ...', chatbot, history, delay=0) # 刷新界面
-
         if len(txt_origin) > REMEMBER_PREVIEW:
             HALF = REMEMBER_PREVIEW//2
             i_say_to_remember = txt[:HALF] + f" ...\n...(省略{len(txt_origin)-REMEMBER_PREVIEW}字)...\n... " + txt[-HALF:]
             if (flags["original_input_len"] - flags["clipped_input_len"]) > HALF:
                 txt_clip = txt_clip  + f" ...\n...(省略{len(txt_origin)-len(txt_clip)-HALF}字)...\n... " + txt[-HALF:]
+            else:
+                pass
+            i_say = txt_clip
         else:
             i_say_to_remember = i_say = txt_clip
     else:
         i_say_to_remember = i_say = txt_clip
 
-    # 6. Search vector store and build prompts
+    # 3. we search vector store and build prompts
     nodes = rag_worker.retrieve_from_store_with_query(i_say)
     prompt = rag_worker.build_prompt(query=i_say, nodes=nodes)
-    # 7. Query language model
-    if len(chatbot) != 0:
-        chatbot.pop(-1)  # Pop temp chat, because we are going to add them again inside `request_gpt_model_in_new_thread_with_ui_alive`
 
+    # 4. it is time to query llms
+    if len(chatbot) != 0: chatbot.pop(-1) # pop temp chat, because we are going to add them again inside `request_gpt_model_in_new_thread_with_ui_alive`
     model_say = yield from request_gpt_model_in_new_thread_with_ui_alive(
-        inputs=prompt,
+        inputs=prompt, inputs_show_user=i_say,
-        inputs_show_user=i_say,
+        llm_kwargs=llm_kwargs, chatbot=chatbot, history=history,
-        llm_kwargs=llm_kwargs,
-        chatbot=chatbot,
-        history=history,
         sys_prompt=system_prompt,
         retry_times_at_unknown_error=0
     )
 
-    # 8. Remember Q&A
+    # 5. remember what has been asked / answered
-    yield from update_ui_lastest_msg(
+    yield from update_ui_lastest_msg(model_say + '</br></br>' + f'对话记忆中, 请稍等 ({current_context}) ...', chatbot, history, delay=0.5) # 刷新界面
-        model_say + '</br></br>' + f'对话记忆中, 请稍等 ({current_context}) ...',
-        chatbot, history, delay=0.5
-    )
     rag_worker.remember_qa(i_say_to_remember, model_say)
     history.extend([i_say, model_say])
 
-    # 9. Final UI Update
+    yield from update_ui_lastest_msg(model_say, chatbot, history, delay=0, msg=tip) # 刷新界面
-    yield from update_ui_lastest_msg(model_say, chatbot, history, delay=0, msg=tip)

crazy_functions/latex_fns/latex_actions.py

@@ -3,7 +3,7 @@ import re
 import shutil
 import numpy as np
 from loguru import logger
-from toolbox import update_ui, update_ui_lastest_msg, get_log_folder, gen_time_str
+from toolbox import update_ui, update_ui_lastest_msg, get_log_folder
 from toolbox import get_conf, promote_file_to_downloadzone
 from crazy_functions.latex_fns.latex_toolbox import PRESERVE, TRANSFORM
 from crazy_functions.latex_fns.latex_toolbox import set_forbidden_text, set_forbidden_text_begin_end, set_forbidden_text_careful_brace
@@ -423,9 +423,6 @@ def 编译Latex(chatbot, history, main_file_original, main_file_modified, work_f
                 except Exception as e:
                     logger.error(e)
                     pass
-
-
-
             return True # 成功啦
         else:
             if n_fix>=max_try: break
@@ -471,66 +468,3 @@ def write_html(sp_file_contents, sp_file_result, chatbot, project_folder):
     except:
         from toolbox import trimmed_format_exc
         logger.error('writing html result failed:', trimmed_format_exc())
-
-
-def upload_to_gptac_cloud_if_user_allow(chatbot, arxiv_id):
-    try:
-        # 如果用户允许，我们将arxiv论文PDF上传到GPTAC学术云
-        from toolbox import map_file_to_sha256
-        # 检查是否顺利，如果没有生成预期的文件，则跳过
-        is_result_good = False
-        for file_path in chatbot._cookies.get("files_to_promote", []):
-            if file_path.endswith('translate_zh.pdf'):
-                is_result_good = True
-        if not is_result_good:
-            return
-        # 上传文件
-        for file_path in chatbot._cookies.get("files_to_promote", []):
-            align_name = None
-            # normalized name
-            for name in ['translate_zh.pdf', 'comparison.pdf']:
-                if file_path.endswith(name): align_name = name
-            # if match any align name
-            if align_name:
-                logger.info(f'Uploading to GPTAC cloud as the user has set `allow_cloud_io`: {file_path}')
-                with open(file_path, 'rb') as f:
-                    import requests
-                    url = 'https://cloud-2.agent-matrix.com/upload'
-                    files = {'file': (align_name, f, 'application/octet-stream')}
-                    data = {
-                        'arxiv_id': arxiv_id,
-                        'file_hash': map_file_to_sha256(file_path),
-                    }
-                    resp = requests.post(url=url, files=files, data=data, timeout=30)
-                logger.info(f'Uploading terminate ({resp.status_code})`: {file_path}')
-    except:
-        # 如果上传失败，不会中断程序，因为这是次要功能
-        pass
-
-def check_gptac_cloud(arxiv_id, chatbot):
-    import requests
-    success = False
-    downloaded = []
-    try:
-        for pdf_target in ['translate_zh.pdf', 'comparison.pdf']:
-            url = 'https://cloud-2.agent-matrix.com/paper_exist'
-            data = {
-                'arxiv_id': arxiv_id,
-                'name': pdf_target,
-            }
-            resp = requests.post(url=url, data=data)
-            cache_hit_result = resp.text.strip('"')
-            if cache_hit_result.startswith("http"):
-                url = cache_hit_result
-                logger.info(f'Downloading from GPTAC cloud: {url}')
-                resp = requests.get(url=url, timeout=30)
-                target = os.path.join(get_log_folder(plugin_name='gptac_cloud'), gen_time_str(), pdf_target)
-                os.makedirs(os.path.dirname(target), exist_ok=True)
-                with open(target, 'wb') as f:
-                    f.write(resp.content)
-                new_path = promote_file_to_downloadzone(target, chatbot=chatbot)
-                success = True
-                downloaded.append(new_path)
-    except:
-        pass
-    return success, downloaded

crazy_functions/latex_fns/latex_toolbox.py

@@ -697,6 +697,15 @@ def _merge_pdfs_ng(pdf1_path, pdf2_path, output_path):
                     ),
                     0,
                 )
+                if "/Annots" in page1:
+                    page1_annot_id = [annot.idnum for annot in page1["/Annots"]]
+                else:
+                    page1_annot_id = []
+
+                if "/Annots" in page2:
+                    page2_annot_id = [annot.idnum for annot in page2["/Annots"]]
+                else:
+                    page2_annot_id = []
                 if "/Annots" in new_page:
                     annotations = new_page["/Annots"]
                     for i, annot in enumerate(annotations):
@@ -711,10 +720,7 @@ def _merge_pdfs_ng(pdf1_path, pdf2_path, output_path):
                                 if "/S" in action and action["/S"] == "/GoTo":
                                     # 内部链接：跳转到文档中的某个页面
                                     dest = action.get("/D")  # 目标页或目标位置
-                                    # if dest and annot.idnum in page2_annot_id:
+                                    if dest and annot.idnum in page2_annot_id:
-                                    # if dest in pdf2_reader.named_destinations:
-                                    if dest and page2.annotations:
-                                        if annot in page2.annotations:
                                         # 获取原始文件中跳转信息，包括跳转页面
                                         destination = pdf2_reader.named_destinations[
                                             dest
@@ -726,7 +732,6 @@ def _merge_pdfs_ng(pdf1_path, pdf2_path, output_path):
                                         )
                                         # 更新跳转信息，跳转到对应的页面和，指定坐标 (100, 150)，缩放比例为 100%
                                         # “/D”:[10,'/XYZ',100,100,0]
-                                            if destination.dest_array[1] == "/XYZ":
                                         annot_obj["/A"].update(
                                             {
                                                 NameObject("/D"): ArrayObject(
@@ -734,9 +739,7 @@ def _merge_pdfs_ng(pdf1_path, pdf2_path, output_path):
                                                         NumberObject(page_number),
                                                         destination.dest_array[1],
                                                         FloatObject(
-                                                                    destination.dest_array[
+                                                            destination.dest_array[2]
-                                                                        2
-                                                                    ]
                                                             + int(
                                                                 page1.mediaBox.getWidth()
                                                             )
@@ -747,18 +750,6 @@ def _merge_pdfs_ng(pdf1_path, pdf2_path, output_path):
                                                 )  # 确保键和值是 PdfObject
                                             }
                                         )
-                                            else:
-                                                annot_obj["/A"].update(
-                                                    {
-                                                        NameObject("/D"): ArrayObject(
-                                                            [
-                                                                NumberObject(page_number),
-                                                                destination.dest_array[1],
-                                                            ]
-                                                        )  # 确保键和值是 PdfObject
-                                                    }
-                                                )
-
                                         rect = annot_obj.get("/Rect")
                                         # 更新点击坐标
                                         rect = ArrayObject(
@@ -782,10 +773,7 @@ def _merge_pdfs_ng(pdf1_path, pdf2_path, output_path):
                                                 ): rect  # 确保键和值是 PdfObject
                                             }
                                         )
-                                    # if dest and annot.idnum in page1_annot_id:
+                                    if dest and annot.idnum in page1_annot_id:
-                                    # if dest in pdf1_reader.named_destinations:
-                                    if dest and page1.annotations:
-                                        if annot in page1.annotations:
                                         # 获取原始文件中跳转信息，包括跳转页面
                                         destination = pdf1_reader.named_destinations[
                                             dest
@@ -797,7 +785,6 @@ def _merge_pdfs_ng(pdf1_path, pdf2_path, output_path):
                                         )
                                         # 更新跳转信息，跳转到对应的页面和，指定坐标 (100, 150)，缩放比例为 100%
                                         # “/D”:[10,'/XYZ',100,100,0]
-                                            if destination.dest_array[1] == "/XYZ":
                                         annot_obj["/A"].update(
                                             {
                                                 NameObject("/D"): ArrayObject(
@@ -805,9 +792,7 @@ def _merge_pdfs_ng(pdf1_path, pdf2_path, output_path):
                                                         NumberObject(page_number),
                                                         destination.dest_array[1],
                                                         FloatObject(
-                                                                    destination.dest_array[
+                                                            destination.dest_array[2]
-                                                                        2
-                                                                    ]
                                                         ),
                                                         destination.dest_array[3],
                                                         destination.dest_array[4],
@@ -815,18 +800,6 @@ def _merge_pdfs_ng(pdf1_path, pdf2_path, output_path):
                                                 )  # 确保键和值是 PdfObject
                                             }
                                         )
-                                            else:
-                                                annot_obj["/A"].update(
-                                                    {
-                                                        NameObject("/D"): ArrayObject(
-                                                            [
-                                                                NumberObject(page_number),
-                                                                destination.dest_array[1],
-                                                            ]
-                                                        )  # 确保键和值是 PdfObject
-                                                    }
-                                                )
-
                                         rect = annot_obj.get("/Rect")
                                         rect = ArrayObject(
                                             [
@@ -847,12 +820,14 @@ def _merge_pdfs_ng(pdf1_path, pdf2_path, output_path):
                                 elif "/S" in action and action["/S"] == "/URI":
                                     # 外部链接：跳转到某个URI
                                     uri = action.get("/URI")
+
                 output_writer.addPage(new_page)
             # Save the merged PDF file
             with open(output_path, "wb") as output_file:
                 output_writer.write(output_file)
 
 
+
 def _merge_pdfs_legacy(pdf1_path, pdf2_path, output_path):
     import PyPDF2  # PyPDF2这个库有严重的内存泄露问题，把它放到子进程中运行，从而方便内存的释放
 

crazy_functions/rag_fns/llama_index_worker.py

@@ -1,13 +1,17 @@
+import llama_index
+import os
 import atexit
 from loguru import logger
 from typing import List
-
 from llama_index.core import Document
-from llama_index.core.ingestion import run_transformations
 from llama_index.core.schema import TextNode
-
-from crazy_functions.rag_fns.vector_store_index import GptacVectorStoreIndex
 from request_llms.embed_models.openai_embed import OpenAiEmbeddingModel
+from shared_utils.connect_void_terminal import get_chat_default_kwargs
+from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
+from crazy_functions.rag_fns.vector_store_index import GptacVectorStoreIndex
+from llama_index.core.ingestion import run_transformations
+from llama_index.core import PromptTemplate
+from llama_index.core.response_synthesizers import TreeSummarize
 
 DEFAULT_QUERY_GENERATION_PROMPT = """\
 Now, you have context information as below:
@@ -59,7 +63,7 @@ class SaveLoad():
     def purge(self):
         import shutil
         shutil.rmtree(self.checkpoint_dir, ignore_errors=True)
-        self.vs_index = self.create_new_vs(self.checkpoint_dir)
+        self.vs_index = self.create_new_vs()
 
 
 class LlamaIndexRagWorker(SaveLoad):
@@ -71,7 +75,7 @@ class LlamaIndexRagWorker(SaveLoad):
         if auto_load_checkpoint:
             self.vs_index = self.load_from_checkpoint(checkpoint_dir)
         else:
-            self.vs_index = self.create_new_vs()
+            self.vs_index = self.create_new_vs(checkpoint_dir)
         atexit.register(lambda: self.save_to_checkpoint(checkpoint_dir))
 
     def assign_embedding_model(self):
@@ -87,21 +91,17 @@ class LlamaIndexRagWorker(SaveLoad):
         logger.info('oo --------inspect_vector_store end--------')
         return vector_store_preview
 
-    def add_documents_to_vector_store(self, document_list: List[Document]):
+    def add_documents_to_vector_store(self, document_list):
-        """
+        documents = [Document(text=t) for t in document_list]
-        Adds a list of Document objects to the vector store after processing.
-        """
-        documents = document_list
         documents_nodes = run_transformations(
                         documents,  # type: ignore
                         self.vs_index._transformations,
                         show_progress=True
                     )
         self.vs_index.insert_nodes(documents_nodes)
-        if self.debug_mode:
+        if self.debug_mode: self.inspect_vector_store()
-            self.inspect_vector_store()
 
-    def add_text_to_vector_store(self, text: str):
+    def add_text_to_vector_store(self, text):
         node = TextNode(text=text)
         documents_nodes = run_transformations(
                         [node],
@@ -109,16 +109,14 @@ class LlamaIndexRagWorker(SaveLoad):
                         show_progress=True
                     )
         self.vs_index.insert_nodes(documents_nodes)
-        if self.debug_mode:
+        if self.debug_mode: self.inspect_vector_store()
-            self.inspect_vector_store()
 
     def remember_qa(self, question, answer):
         formatted_str = QUESTION_ANSWER_RECORD.format(question=question, answer=answer)
         self.add_text_to_vector_store(formatted_str)
 
     def retrieve_from_store_with_query(self, query):
-        if self.debug_mode:
+        if self.debug_mode: self.inspect_vector_store()
-            self.inspect_vector_store()
         retriever = self.vs_index.as_retriever()
         return retriever.retrieve(query)
 
@@ -130,9 +128,3 @@ class LlamaIndexRagWorker(SaveLoad):
         buf = "\n".join(([f"(No.{i+1} | score {n.score:.3f}): {n.text}" for i, n in enumerate(nodes)]))
         if self.debug_mode: logger.info(buf)
         return buf
-
-    def purge_vector_store(self):
-        """
-        Purges the current vector store and creates a new one.
-        """
-        self.purge()

crazy_functions/rag_fns/rag_file_support.py

@@ -1,22 +0,0 @@
-import os
-from llama_index.core import SimpleDirectoryReader
-
-supports_format = ['.csv', '.docx', '.epub', '.ipynb',  '.mbox', '.md', '.pdf',  '.txt', '.ppt',
-                   '.pptm', '.pptx']
-
-
-# 修改后的 extract_text 函数，结合 SimpleDirectoryReader 和自定义解析逻辑
-def extract_text(file_path):
-    _, ext = os.path.splitext(file_path.lower())
-
-    # 使用 SimpleDirectoryReader 处理它支持的文件格式
-    if ext in supports_format:
-        try:
-            reader = SimpleDirectoryReader(input_files=[file_path])
-            documents = reader.load_data()
-            if len(documents) > 0:
-                return documents[0].text
-        except Exception as e:
-            pass
-
-    return None

docs/GithubAction+NoLocal+Latex+Arm

@@ -1,25 +0,0 @@
-# 此Dockerfile适用于“无本地模型”的环境构建，如果需要使用chatglm等本地模型，请参考 docs/Dockerfile+ChatGLM
-# - 1 修改 `config.py`
-# - 2 构建 docker build -t gpt-academic-nolocal-latex -f docs/GithubAction+NoLocal+Latex .
-# - 3 运行 docker run -v /home/fuqingxu/arxiv_cache:/root/arxiv_cache --rm -it --net=host gpt-academic-nolocal-latex
-
-FROM menghuan1918/ubuntu_uv_ctex:latest
-ENV DEBIAN_FRONTEND=noninteractive
-SHELL ["/bin/bash", "-c"]
-WORKDIR /gpt
-COPY . .
-RUN /root/.cargo/bin/uv venv --seed \
-    && source .venv/bin/activate \
-    && /root/.cargo/bin/uv pip install openai numpy arxiv rich colorama Markdown pygments pymupdf python-docx pdfminer \
-    && /root/.cargo/bin/uv pip install -r requirements.txt \
-    && /root/.cargo/bin/uv clean
-
-# 对齐python3
-RUN rm -f /usr/bin/python3 && ln -s /gpt/.venv/bin/python /usr/bin/python3
-RUN rm -f /usr/bin/python && ln -s /gpt/.venv/bin/python /usr/bin/python
-
-# 可选步骤，用于预热模块
-RUN python3 -c 'from check_proxy import warm_up_modules; warm_up_modules()'
-
-# 启动
-CMD ["python3", "-u", "main.py"]

request_llms/bridge_all.py

@@ -256,8 +256,6 @@ model_info = {
         "max_token": 128000,
         "tokenizer": tokenizer_gpt4,
         "token_cnt": get_token_num_gpt4,
-        "openai_disable_system_prompt": True,
-        "openai_disable_stream": True,
     },
     "o1-mini": {
         "fn_with_ui": chatgpt_ui,
@@ -266,8 +264,6 @@ model_info = {
         "max_token": 128000,
         "tokenizer": tokenizer_gpt4,
         "token_cnt": get_token_num_gpt4,
-        "openai_disable_system_prompt": True,
-        "openai_disable_stream": True,
     },
 
     "gpt-4-turbo": {
@@ -1285,3 +1281,4 @@ def predict(inputs:str, llm_kwargs:dict, plugin_kwargs:dict, chatbot,
 
     # 更新一下llm_kwargs的参数，否则会出现参数不匹配的问题
     yield from method(inputs, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, stream, additional_fn)
+

request_llms/bridge_chatgpt.py

@@ -202,13 +202,10 @@ def predict_no_ui_long_connection(inputs:str, llm_kwargs:dict, history:list=[],
                     if (time.time()-observe_window[1]) > watch_dog_patience:
                         raise RuntimeError("用户取消了程序。")
         else: raise RuntimeError("意外Json结构："+delta)
-
+    if json_data and json_data['finish_reason'] == 'content_filter':
-    finish_reason = json_data.get('finish_reason', None) if json_data else None
+        raise RuntimeError("由于提问含不合规内容被Azure过滤。")
-    if finish_reason == 'content_filter':
+    if json_data and json_data['finish_reason'] == 'length':
-        raise RuntimeError("由于提问含不合规内容被过滤。")
-    if finish_reason == 'length':
         raise ConnectionAbortedError("正常结束，但显示Token不足，导致输出不完整，请削减单次输入的文本量。")
-
     return result
 
 
@@ -539,3 +536,4 @@ def generate_payload(inputs:str, llm_kwargs:dict, history:list, system_prompt:st
 
     return headers,payload
 
+

tests/test_latex_auto_correct.py

@@ -19,8 +19,4 @@ if __name__ == "__main__":
     plugin_test = importlib.import_module('test_utils').plugin_test
 
 
-    # plugin_test(plugin='crazy_functions.Latex_Function->Latex翻译中文并重新编译PDF', main_input="2203.01927")
+    plugin_test(plugin='crazy_functions.Latex_Function->Latex翻译中文并重新编译PDF', main_input="2203.01927")
-    # plugin_test(plugin='crazy_functions.Latex_Function->Latex翻译中文并重新编译PDF', main_input="gpt_log/arxiv_cache/2203.01927/workfolder")
-    # plugin_test(plugin='crazy_functions.Latex_Function->Latex翻译中文并重新编译PDF', main_input="2410.05779")
-    plugin_test(plugin='crazy_functions.Latex_Function->Latex翻译中文并重新编译PDF', main_input="gpt_log/default_user/workfolder")
-