镜像自地址
https://github.com/binary-husky/gpt_academic.git
已同步 2025-12-06 22:46:48 +00:00
把上传文件路径和日志路径修改为统一可配置的变量
这个提交包含在:
@@ -1,6 +1,7 @@
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from typing import Any
|
||||
from toolbox import CatchException, update_ui, gen_time_str, trimmed_format_exc, promote_file_to_downloadzone, clear_file_downloadzone
|
||||
from toolbox import CatchException, update_ui, gen_time_str, trimmed_format_exc
|
||||
from toolbox import promote_file_to_downloadzone, get_log_folder
|
||||
from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||
from .crazy_utils import input_clipping, try_install_deps
|
||||
from multiprocessing import Process, Pipe
|
||||
@@ -92,7 +93,7 @@ def gpt_interact_multi_step(txt, file_type, llm_kwargs, chatbot, history):
|
||||
|
||||
def make_module(code):
|
||||
module_file = 'gpt_fn_' + gen_time_str().replace('-','_')
|
||||
with open(f'gpt_log/{module_file}.py', 'w', encoding='utf8') as f:
|
||||
with open(f'{get_log_folder()}/{module_file}.py', 'w', encoding='utf8') as f:
|
||||
f.write(code)
|
||||
|
||||
def get_class_name(class_string):
|
||||
@@ -102,7 +103,7 @@ def make_module(code):
|
||||
return class_name
|
||||
|
||||
class_name = get_class_name(code)
|
||||
return f"gpt_log.{module_file}->{class_name}"
|
||||
return f"{get_log_folder().replace('/', '.')}.{module_file}->{class_name}"
|
||||
|
||||
def init_module_instance(module):
|
||||
import importlib
|
||||
@@ -171,7 +172,7 @@ def 虚空终端CodeInterpreter(txt, llm_kwargs, plugin_kwargs, chatbot, history
|
||||
file_type = file_path.split('.')[-1]
|
||||
|
||||
# 粗心检查
|
||||
if 'private_upload' in txt:
|
||||
if is_the_upload_folder(txt):
|
||||
chatbot.append([
|
||||
"...",
|
||||
f"请在输入框内填写需求,然后再次点击该插件(文件路径 {file_path} 已经被记忆)"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from toolbox import update_ui, trimmed_format_exc
|
||||
from toolbox import CatchException, report_execption, write_results_to_file, zip_folder
|
||||
from toolbox import update_ui, trimmed_format_exc, promote_file_to_downloadzone, get_log_folder
|
||||
from toolbox import CatchException, report_execption, write_history_to_file, zip_folder
|
||||
|
||||
|
||||
class PaperFileGroup():
|
||||
@@ -51,7 +51,7 @@ class PaperFileGroup():
|
||||
import os, time
|
||||
folder = os.path.dirname(self.file_paths[0])
|
||||
t = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
|
||||
zip_folder(folder, './gpt_log/', f'{t}-polished.zip')
|
||||
zip_folder(folder, get_log_folder(), f'{t}-polished.zip')
|
||||
|
||||
|
||||
def 多文件润色(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en', mode='polish'):
|
||||
@@ -126,7 +126,9 @@ def 多文件润色(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch
|
||||
|
||||
# <-------- 整理结果,退出 ---------->
|
||||
create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md"
|
||||
res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name)
|
||||
res = write_history_to_file(gpt_response_collection, file_basename=create_report_file_name)
|
||||
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||
|
||||
history = gpt_response_collection
|
||||
chatbot.append((f"{fp}完成了吗?", res))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from toolbox import update_ui
|
||||
from toolbox import CatchException, report_execption, write_results_to_file
|
||||
from toolbox import update_ui, promote_file_to_downloadzone
|
||||
from toolbox import CatchException, report_execption, write_history_to_file
|
||||
fast_debug = False
|
||||
|
||||
class PaperFileGroup():
|
||||
@@ -95,7 +95,8 @@ def 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, ch
|
||||
|
||||
# <-------- 整理结果,退出 ---------->
|
||||
create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md"
|
||||
res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name)
|
||||
res = write_history_to_file(gpt_response_collection, create_report_file_name)
|
||||
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||
history = gpt_response_collection
|
||||
chatbot.append((f"{fp}完成了吗?", res))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from toolbox import update_ui, trimmed_format_exc, get_conf, objdump, objload, promote_file_to_downloadzone
|
||||
from toolbox import update_ui, trimmed_format_exc, get_conf, get_log_folder, promote_file_to_downloadzone
|
||||
from toolbox import CatchException, report_execption, update_ui_lastest_msg, zip_result, gen_time_str
|
||||
from functools import partial
|
||||
import glob, os, requests, time
|
||||
@@ -65,7 +65,7 @@ def move_project(project_folder, arxiv_id=None):
|
||||
if arxiv_id is not None:
|
||||
new_workfolder = pj(ARXIV_CACHE_DIR, arxiv_id, 'workfolder')
|
||||
else:
|
||||
new_workfolder = f'gpt_log/{gen_time_str()}'
|
||||
new_workfolder = f'{get_log_folder()}/{gen_time_str()}'
|
||||
try:
|
||||
shutil.rmtree(new_workfolder)
|
||||
except:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from toolbox import update_ui, update_ui_lastest_msg # 刷新Gradio前端界面
|
||||
from toolbox import update_ui, update_ui_lastest_msg, get_log_folder
|
||||
from toolbox import zip_folder, objdump, objload, promote_file_to_downloadzone
|
||||
from .latex_toolbox import PRESERVE, TRANSFORM
|
||||
from .latex_toolbox import set_forbidden_text, set_forbidden_text_begin_end, set_forbidden_text_careful_brace
|
||||
@@ -439,9 +439,9 @@ def write_html(sp_file_contents, sp_file_result, chatbot, project_folder):
|
||||
trans = k
|
||||
ch.add_row(a=orig, b=trans)
|
||||
create_report_file_name = f"{gen_time_str()}.trans.html"
|
||||
ch.save_file(create_report_file_name)
|
||||
shutil.copyfile(pj('./gpt_log/', create_report_file_name), pj(project_folder, create_report_file_name))
|
||||
promote_file_to_downloadzone(file=f'./gpt_log/{create_report_file_name}', chatbot=chatbot)
|
||||
res = ch.save_file(create_report_file_name)
|
||||
shutil.copyfile(res, pj(project_folder, create_report_file_name))
|
||||
promote_file_to_downloadzone(file=res, chatbot=chatbot)
|
||||
except:
|
||||
from toolbox import trimmed_format_exc
|
||||
print('writing html result failed:', trimmed_format_exc())
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from toolbox import update_ui
|
||||
from toolbox import CatchException, report_execption, write_results_to_file, get_conf
|
||||
from toolbox import update_ui, get_log_folder
|
||||
from toolbox import write_history_to_file, promote_file_to_downloadzone
|
||||
from toolbox import CatchException, report_execption, get_conf
|
||||
import re, requests, unicodedata, os
|
||||
from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||
def download_arxiv_(url_pdf):
|
||||
@@ -28,7 +29,7 @@ def download_arxiv_(url_pdf):
|
||||
if k in other_info['comment']:
|
||||
title = k + ' ' + title
|
||||
|
||||
download_dir = './gpt_log/arxiv/'
|
||||
download_dir = get_log_folder(plugin_name='arxiv')
|
||||
os.makedirs(download_dir, exist_ok=True)
|
||||
|
||||
title_str = title.replace('?', '?')\
|
||||
@@ -40,9 +41,6 @@ def download_arxiv_(url_pdf):
|
||||
|
||||
requests_pdf_url = url_pdf
|
||||
file_path = download_dir+title_str
|
||||
# if os.path.exists(file_path):
|
||||
# print('返回缓存文件')
|
||||
# return './gpt_log/arxiv/'+title_str
|
||||
|
||||
print('下载中')
|
||||
proxies, = get_conf('proxies')
|
||||
@@ -61,7 +59,7 @@ def download_arxiv_(url_pdf):
|
||||
.replace('\n', '')\
|
||||
.replace(' ', ' ')\
|
||||
.replace(' ', ' ')
|
||||
return './gpt_log/arxiv/'+title_str, other_info
|
||||
return file_path, other_info
|
||||
|
||||
|
||||
def get_name(_url_):
|
||||
@@ -184,11 +182,10 @@ def 下载arxiv论文并翻译摘要(txt, llm_kwargs, plugin_kwargs, chatbot, hi
|
||||
chatbot[-1] = (i_say_show_user, gpt_say)
|
||||
history.append(i_say_show_user); history.append(gpt_say)
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面
|
||||
# 写入文件
|
||||
import shutil
|
||||
# 重置文件的创建时间
|
||||
shutil.copyfile(pdf_path, f'./gpt_log/{os.path.basename(pdf_path)}'); os.remove(pdf_path)
|
||||
res = write_results_to_file(history)
|
||||
res = write_history_to_file(history)
|
||||
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||
promote_file_to_downloadzone(pdf_path, chatbot=chatbot)
|
||||
|
||||
chatbot.append(("完成了吗?", res + "\n\nPDF文件也已经下载"))
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面
|
||||
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
import threading
|
||||
from request_llm.bridge_all import predict_no_ui_long_connection
|
||||
from toolbox import update_ui
|
||||
from toolbox import CatchException, write_results_to_file, report_execption
|
||||
from .crazy_utils import breakdown_txt_to_satisfy_token_limit
|
||||
|
||||
def extract_code_block_carefully(txt):
|
||||
splitted = txt.split('```')
|
||||
n_code_block_seg = len(splitted) - 1
|
||||
if n_code_block_seg <= 1: return txt
|
||||
# 剩下的情况都开头除去 ``` 结尾除去一次 ```
|
||||
txt_out = '```'.join(splitted[1:-1])
|
||||
return txt_out
|
||||
|
||||
|
||||
|
||||
def break_txt_into_half_at_some_linebreak(txt):
|
||||
lines = txt.split('\n')
|
||||
n_lines = len(lines)
|
||||
pre = lines[:(n_lines//2)]
|
||||
post = lines[(n_lines//2):]
|
||||
return "\n".join(pre), "\n".join(post)
|
||||
|
||||
|
||||
@CatchException
|
||||
def 全项目切换英文(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys_prompt, web_port):
|
||||
# 第1步:清空历史,以免输入溢出
|
||||
history = []
|
||||
|
||||
# 第2步:尝试导入依赖,如果缺少依赖,则给出安装建议
|
||||
try:
|
||||
import tiktoken
|
||||
except:
|
||||
report_execption(chatbot, history,
|
||||
a = f"解析项目: {txt}",
|
||||
b = f"导入软件依赖失败。使用该模块需要额外依赖,安装方法```pip install --upgrade tiktoken```。")
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
return
|
||||
|
||||
# 第3步:集合文件
|
||||
import time, glob, os, shutil, re
|
||||
os.makedirs('gpt_log/generated_english_version', exist_ok=True)
|
||||
os.makedirs('gpt_log/generated_english_version/crazy_functions', exist_ok=True)
|
||||
file_manifest = [f for f in glob.glob('./*.py') if ('test_project' not in f) and ('gpt_log' not in f)] + \
|
||||
[f for f in glob.glob('./crazy_functions/*.py') if ('test_project' not in f) and ('gpt_log' not in f)]
|
||||
# file_manifest = ['./toolbox.py']
|
||||
i_say_show_user_buffer = []
|
||||
|
||||
# 第4步:随便显示点什么防止卡顿的感觉
|
||||
for index, fp in enumerate(file_manifest):
|
||||
# if 'test_project' in fp: continue
|
||||
with open(fp, 'r', encoding='utf-8', errors='replace') as f:
|
||||
file_content = f.read()
|
||||
i_say_show_user =f'[{index}/{len(file_manifest)}] 接下来请将以下代码中包含的所有中文转化为英文,只输出转化后的英文代码,请用代码块输出代码: {os.path.abspath(fp)}'
|
||||
i_say_show_user_buffer.append(i_say_show_user)
|
||||
chatbot.append((i_say_show_user, "[Local Message] 等待多线程操作,中间过程不予显示."))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
|
||||
# 第5步:Token限制下的截断与处理
|
||||
MAX_TOKEN = 3000
|
||||
from request_llm.bridge_all import model_info
|
||||
enc = model_info["gpt-3.5-turbo"]['tokenizer']
|
||||
def get_token_fn(txt): return len(enc.encode(txt, disallowed_special=()))
|
||||
|
||||
|
||||
# 第6步:任务函数
|
||||
mutable_return = [None for _ in file_manifest]
|
||||
observe_window = [[""] for _ in file_manifest]
|
||||
def thread_worker(fp,index):
|
||||
if index > 10:
|
||||
time.sleep(60)
|
||||
print('Openai 限制免费用户每分钟20次请求,降低请求频率中。')
|
||||
with open(fp, 'r', encoding='utf-8', errors='replace') as f:
|
||||
file_content = f.read()
|
||||
i_say_template = lambda fp, file_content: f'接下来请将以下代码中包含的所有中文转化为英文,只输出代码,文件名是{fp},文件代码是 ```{file_content}```'
|
||||
try:
|
||||
gpt_say = ""
|
||||
# 分解代码文件
|
||||
file_content_breakdown = breakdown_txt_to_satisfy_token_limit(file_content, get_token_fn, MAX_TOKEN)
|
||||
for file_content_partial in file_content_breakdown:
|
||||
i_say = i_say_template(fp, file_content_partial)
|
||||
# # ** gpt request **
|
||||
gpt_say_partial = predict_no_ui_long_connection(inputs=i_say, llm_kwargs=llm_kwargs, history=[], sys_prompt=sys_prompt, observe_window=observe_window[index])
|
||||
gpt_say_partial = extract_code_block_carefully(gpt_say_partial)
|
||||
gpt_say += gpt_say_partial
|
||||
mutable_return[index] = gpt_say
|
||||
except ConnectionAbortedError as token_exceed_err:
|
||||
print('至少一个线程任务Token溢出而失败', e)
|
||||
except Exception as e:
|
||||
print('至少一个线程任务意外失败', e)
|
||||
|
||||
# 第7步:所有线程同时开始执行任务函数
|
||||
handles = [threading.Thread(target=thread_worker, args=(fp,index)) for index, fp in enumerate(file_manifest)]
|
||||
for h in handles:
|
||||
h.daemon = True
|
||||
h.start()
|
||||
chatbot.append(('开始了吗?', f'多线程操作已经开始'))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
# 第8步:循环轮询各个线程是否执行完毕
|
||||
cnt = 0
|
||||
while True:
|
||||
cnt += 1
|
||||
time.sleep(0.2)
|
||||
th_alive = [h.is_alive() for h in handles]
|
||||
if not any(th_alive): break
|
||||
# 更好的UI视觉效果
|
||||
observe_win = []
|
||||
for thread_index, alive in enumerate(th_alive):
|
||||
observe_win.append("[ ..."+observe_window[thread_index][0][-60:].replace('\n','').replace('```','...').replace(' ','.').replace('<br/>','.....').replace('$','.')+"... ]")
|
||||
stat = [f'执行中: {obs}\n\n' if alive else '已完成\n\n' for alive, obs in zip(th_alive, observe_win)]
|
||||
stat_str = ''.join(stat)
|
||||
chatbot[-1] = (chatbot[-1][0], f'多线程操作已经开始,完成情况: \n\n{stat_str}' + ''.join(['.']*(cnt%10+1)))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
# 第9步:把结果写入文件
|
||||
for index, h in enumerate(handles):
|
||||
h.join() # 这里其实不需要join了,肯定已经都结束了
|
||||
fp = file_manifest[index]
|
||||
gpt_say = mutable_return[index]
|
||||
i_say_show_user = i_say_show_user_buffer[index]
|
||||
|
||||
where_to_relocate = f'gpt_log/generated_english_version/{fp}'
|
||||
if gpt_say is not None:
|
||||
with open(where_to_relocate, 'w+', encoding='utf-8') as f:
|
||||
f.write(gpt_say)
|
||||
else: # 失败
|
||||
shutil.copyfile(file_manifest[index], where_to_relocate)
|
||||
chatbot.append((i_say_show_user, f'[Local Message] 已完成{os.path.abspath(fp)}的转化,\n\n存入{os.path.abspath(where_to_relocate)}'))
|
||||
history.append(i_say_show_user); history.append(gpt_say)
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
time.sleep(1)
|
||||
|
||||
# 第10步:备份一个文件
|
||||
res = write_results_to_file(history)
|
||||
chatbot.append(("生成一份任务执行报告", res))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
@@ -1,4 +1,4 @@
|
||||
from toolbox import CatchException, update_ui, get_conf, select_api_key
|
||||
from toolbox import CatchException, update_ui, get_conf, select_api_key, get_log_folder
|
||||
from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||
import datetime
|
||||
|
||||
@@ -33,7 +33,7 @@ def gen_image(llm_kwargs, prompt, resolution="256x256"):
|
||||
raise RuntimeError(response.content.decode())
|
||||
# 文件保存到本地
|
||||
r = requests.get(image_url, proxies=proxies)
|
||||
file_path = 'gpt_log/image_gen/'
|
||||
file_path = f'{get_log_folder()}/image_gen/'
|
||||
os.makedirs(file_path, exist_ok=True)
|
||||
file_name = 'Image' + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '.png'
|
||||
with open(file_path+file_name, 'wb+') as f: f.write(r.content)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from toolbox import CatchException, update_ui, promote_file_to_downloadzone
|
||||
from toolbox import CatchException, update_ui, promote_file_to_downloadzone, get_log_folder
|
||||
from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||
import re
|
||||
|
||||
@@ -10,8 +10,8 @@ def write_chat_to_file(chatbot, history=None, file_name=None):
|
||||
import time
|
||||
if file_name is None:
|
||||
file_name = 'chatGPT对话历史' + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '.html'
|
||||
os.makedirs('./gpt_log/', exist_ok=True)
|
||||
with open(f'./gpt_log/{file_name}', 'w', encoding='utf8') as f:
|
||||
fp = os.path.join(get_log_folder(), file_name)
|
||||
with open(fp, 'w', encoding='utf8') as f:
|
||||
from themes.theme import advanced_css
|
||||
f.write(f'<!DOCTYPE html><head><meta charset="utf-8"><title>对话历史</title><style>{advanced_css}</style></head>')
|
||||
for i, contents in enumerate(chatbot):
|
||||
@@ -29,8 +29,8 @@ def write_chat_to_file(chatbot, history=None, file_name=None):
|
||||
for h in history:
|
||||
f.write("\n>>>" + h)
|
||||
f.write('</code>')
|
||||
promote_file_to_downloadzone(f'./gpt_log/{file_name}', rename_file=file_name, chatbot=chatbot)
|
||||
return '对话历史写入:' + os.path.abspath(f'./gpt_log/{file_name}')
|
||||
promote_file_to_downloadzone(fp, rename_file=file_name, chatbot=chatbot)
|
||||
return '对话历史写入:' + os.path.abspath(fp)
|
||||
|
||||
def gen_file_preview(file_name):
|
||||
try:
|
||||
@@ -106,7 +106,7 @@ def 载入对话历史存档(txt, llm_kwargs, plugin_kwargs, chatbot, history, s
|
||||
if not success:
|
||||
if txt == "": txt = '空空如也的输入栏'
|
||||
import glob
|
||||
local_history = "<br/>".join(["`"+hide_cwd(f)+f" ({gen_file_preview(f)})"+"`" for f in glob.glob(f'gpt_log/**/chatGPT对话历史*.html', recursive=True)])
|
||||
local_history = "<br/>".join(["`"+hide_cwd(f)+f" ({gen_file_preview(f)})"+"`" for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True)])
|
||||
chatbot.append([f"正在查找对话历史文件(html格式): {txt}", f"找不到任何html文件: {txt}。但本地存储了以下历史文件,您可以将任意一个文件路径粘贴到输入区,然后重试:<br/>{local_history}"])
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
return
|
||||
@@ -132,8 +132,8 @@ def 删除所有本地对话历史记录(txt, llm_kwargs, plugin_kwargs, chatbot
|
||||
"""
|
||||
|
||||
import glob, os
|
||||
local_history = "<br/>".join(["`"+hide_cwd(f)+"`" for f in glob.glob(f'gpt_log/**/chatGPT对话历史*.html', recursive=True)])
|
||||
for f in glob.glob(f'gpt_log/**/chatGPT对话历史*.html', recursive=True):
|
||||
local_history = "<br/>".join(["`"+hide_cwd(f)+"`" for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True)])
|
||||
for f in glob.glob(f'{get_log_folder()}/**/chatGPT对话历史*.html', recursive=True):
|
||||
os.remove(f)
|
||||
chatbot.append([f"删除所有历史对话文件", f"已删除<br/>{local_history}"])
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from toolbox import update_ui
|
||||
from toolbox import CatchException, report_execption, write_results_to_file
|
||||
from toolbox import CatchException, report_execption
|
||||
from toolbox import write_history_to_file, promote_file_to_downloadzone
|
||||
from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||
fast_debug = False
|
||||
|
||||
@@ -71,11 +72,13 @@ def 解析docx(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot
|
||||
history.extend([i_say,gpt_say])
|
||||
this_paper_history.extend([i_say,gpt_say])
|
||||
|
||||
res = write_results_to_file(history)
|
||||
res = write_history_to_file(history)
|
||||
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||
chatbot.append(("完成了吗?", res))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
res = write_results_to_file(history)
|
||||
res = write_history_to_file(history)
|
||||
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||
chatbot.append(("所有文件都总结完成了吗?", res))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from toolbox import CatchException, report_execption, select_api_key, update_ui, write_results_to_file, get_conf
|
||||
from toolbox import CatchException, report_execption, select_api_key, update_ui, get_conf
|
||||
from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||
from toolbox import write_history_to_file, promote_file_to_downloadzone, get_log_folder
|
||||
|
||||
def split_audio_file(filename, split_duration=1000):
|
||||
"""
|
||||
@@ -15,7 +16,7 @@ def split_audio_file(filename, split_duration=1000):
|
||||
"""
|
||||
from moviepy.editor import AudioFileClip
|
||||
import os
|
||||
os.makedirs('gpt_log/mp3/cut/', exist_ok=True) # 创建存储切割音频的文件夹
|
||||
os.makedirs(f"{get_log_folder(plugin_name='audio')}/mp3/cut/", exist_ok=True) # 创建存储切割音频的文件夹
|
||||
|
||||
# 读取音频文件
|
||||
audio = AudioFileClip(filename)
|
||||
@@ -31,8 +32,8 @@ def split_audio_file(filename, split_duration=1000):
|
||||
start_time = split_points[i]
|
||||
end_time = split_points[i + 1]
|
||||
split_audio = audio.subclip(start_time, end_time)
|
||||
split_audio.write_audiofile(f"gpt_log/mp3/cut/{filename[0]}_{i}.mp3")
|
||||
filelist.append(f"gpt_log/mp3/cut/{filename[0]}_{i}.mp3")
|
||||
split_audio.write_audiofile(f"{get_log_folder(plugin_name='audio')}/mp3/cut/{filename[0]}_{i}.mp3")
|
||||
filelist.append(f"{get_log_folder(plugin_name='audio')}/mp3/cut/{filename[0]}_{i}.mp3")
|
||||
|
||||
audio.close()
|
||||
return filelist
|
||||
@@ -52,7 +53,7 @@ def AnalyAudio(parse_prompt, file_manifest, llm_kwargs, chatbot, history):
|
||||
'Authorization': f"Bearer {api_key}"
|
||||
}
|
||||
|
||||
os.makedirs('gpt_log/mp3/', exist_ok=True)
|
||||
os.makedirs(f"{get_log_folder(plugin_name='audio')}/mp3/", exist_ok=True)
|
||||
for index, fp in enumerate(file_manifest):
|
||||
audio_history = []
|
||||
# 提取文件扩展名
|
||||
@@ -60,8 +61,8 @@ def AnalyAudio(parse_prompt, file_manifest, llm_kwargs, chatbot, history):
|
||||
# 提取视频中的音频
|
||||
if ext not in [".mp3", ".wav", ".m4a", ".mpga"]:
|
||||
audio_clip = AudioFileClip(fp)
|
||||
audio_clip.write_audiofile(f'gpt_log/mp3/output{index}.mp3')
|
||||
fp = f'gpt_log/mp3/output{index}.mp3'
|
||||
audio_clip.write_audiofile(f"{get_log_folder(plugin_name='audio')}/mp3/output{index}.mp3")
|
||||
fp = f"{get_log_folder(plugin_name='audio')}/mp3/output{index}.mp3"
|
||||
# 调用whisper模型音频转文字
|
||||
voice = split_audio_file(fp)
|
||||
for j, i in enumerate(voice):
|
||||
@@ -113,18 +114,19 @@ def AnalyAudio(parse_prompt, file_manifest, llm_kwargs, chatbot, history):
|
||||
history=audio_history,
|
||||
sys_prompt="总结文章。"
|
||||
)
|
||||
|
||||
history.extend([i_say, gpt_say])
|
||||
audio_history.extend([i_say, gpt_say])
|
||||
|
||||
res = write_results_to_file(history)
|
||||
res = write_history_to_file(history)
|
||||
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||
chatbot.append((f"第{index + 1}段音频完成了吗?", res))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
# 删除中间文件夹
|
||||
import shutil
|
||||
shutil.rmtree('gpt_log/mp3')
|
||||
res = write_results_to_file(history)
|
||||
shutil.rmtree(f"{get_log_folder(plugin_name='audio')}/mp3")
|
||||
res = write_history_to_file(history)
|
||||
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||
chatbot.append(("所有音频都总结完成了吗?", res))
|
||||
yield from update_ui(chatbot=chatbot, history=history)
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import glob, time, os, re
|
||||
from toolbox import update_ui, trimmed_format_exc, gen_time_str, disable_auto_promotion
|
||||
from toolbox import CatchException, report_execption, write_history_to_file
|
||||
from toolbox import promote_file_to_downloadzone, get_log_folder
|
||||
from toolbox import CatchException, report_execption, get_log_folder
|
||||
from toolbox import write_history_to_file, promote_file_to_downloadzone
|
||||
fast_debug = False
|
||||
|
||||
class PaperFileGroup():
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from toolbox import update_ui, promote_file_to_downloadzone, gen_time_str
|
||||
from toolbox import CatchException, report_execption, write_results_to_file
|
||||
from toolbox import CatchException, report_execption
|
||||
from toolbox import write_history_to_file, promote_file_to_downloadzone
|
||||
from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||
from .crazy_utils import read_and_clean_pdf_text
|
||||
from .crazy_utils import input_clipping
|
||||
@@ -99,8 +100,8 @@ do not have too much repetitive information, numerical values using the original
|
||||
_, final_results = input_clipping("", final_results, max_token_limit=3200)
|
||||
yield from update_ui(chatbot=chatbot, history=final_results) # 注意这里的历史记录被替代了
|
||||
|
||||
res = write_results_to_file(file_write_buffer, file_name=gen_time_str())
|
||||
promote_file_to_downloadzone(res.split('\t')[-1], chatbot=chatbot)
|
||||
res = write_history_to_file(file_write_buffer)
|
||||
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||
yield from update_ui(chatbot=chatbot, history=final_results) # 刷新界面
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from toolbox import update_ui
|
||||
from toolbox import CatchException, report_execption, write_results_to_file
|
||||
from toolbox import CatchException, report_execption
|
||||
from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||
from toolbox import write_history_to_file, promote_file_to_downloadzone
|
||||
|
||||
fast_debug = False
|
||||
|
||||
@@ -115,7 +116,8 @@ def 解析Paper(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo
|
||||
chatbot[-1] = (i_say, gpt_say)
|
||||
history.append(i_say); history.append(gpt_say)
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面
|
||||
res = write_results_to_file(history)
|
||||
res = write_history_to_file(history)
|
||||
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||
chatbot.append(("完成了吗?", res))
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from toolbox import CatchException, report_execption, write_results_to_file
|
||||
from toolbox import CatchException, report_execption, get_log_folder
|
||||
from toolbox import update_ui, promote_file_to_downloadzone, update_ui_lastest_msg, disable_auto_promotion
|
||||
from toolbox import write_history_to_file, get_log_folder
|
||||
from toolbox import write_history_to_file, promote_file_to_downloadzone
|
||||
from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||
from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency
|
||||
from .crazy_utils import read_and_clean_pdf_text
|
||||
@@ -218,10 +218,11 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot,
|
||||
final = ["一、论文概况\n\n---\n\n", paper_meta_info.replace('# ', '### ') + '\n\n---\n\n', "二、论文翻译", ""]
|
||||
final.extend(gpt_response_collection_md)
|
||||
create_report_file_name = f"{os.path.basename(fp)}.trans.md"
|
||||
res = write_results_to_file(final, file_name=create_report_file_name)
|
||||
res = write_history_to_file(final, create_report_file_name)
|
||||
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||
|
||||
# 更新UI
|
||||
generated_conclusion_files.append(f'./gpt_log/{create_report_file_name}')
|
||||
generated_conclusion_files.append(f'{get_log_folder()}/{create_report_file_name}')
|
||||
chatbot.append((f"{fp}完成了吗?", res))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from toolbox import update_ui
|
||||
from toolbox import CatchException, report_execption, write_results_to_file
|
||||
from toolbox import CatchException, report_execption
|
||||
from toolbox import write_history_to_file, promote_file_to_downloadzone
|
||||
from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||
fast_debug = False
|
||||
|
||||
@@ -27,7 +28,8 @@ def 生成函数注释(file_manifest, project_folder, llm_kwargs, plugin_kwargs,
|
||||
if not fast_debug: time.sleep(2)
|
||||
|
||||
if not fast_debug:
|
||||
res = write_results_to_file(history)
|
||||
res = write_history_to_file(history)
|
||||
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||
chatbot.append(("完成了吗?", res))
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ explain_msg = """
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List
|
||||
from toolbox import CatchException, update_ui, gen_time_str
|
||||
from toolbox import CatchException, update_ui, is_the_upload_folder
|
||||
from toolbox import update_ui_lastest_msg, disable_auto_promotion
|
||||
from request_llm.bridge_all import predict_no_ui_long_connection
|
||||
from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||
@@ -112,7 +112,7 @@ def 虚空终端(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt
|
||||
|
||||
# 用简单的关键词检测用户意图
|
||||
is_certain, _ = analyze_intention_with_simple_rules(txt)
|
||||
if txt.startswith('private_upload/') and len(txt) == 34:
|
||||
if is_the_upload_folder(txt):
|
||||
state.set_state(chatbot=chatbot, key='has_provided_explaination', value=False)
|
||||
appendix_msg = "\n\n**很好,您已经上传了文件**,现在请您描述您的需求。"
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from toolbox import update_ui
|
||||
from toolbox import CatchException, report_execption, write_results_to_file
|
||||
from toolbox import CatchException, report_execption
|
||||
from toolbox import write_history_to_file, promote_file_to_downloadzone
|
||||
fast_debug = True
|
||||
|
||||
|
||||
@@ -110,7 +111,8 @@ def ipynb解释(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
# <-------- 写入文件,退出 ---------->
|
||||
res = write_results_to_file(history)
|
||||
res = write_history_to_file(history)
|
||||
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||
chatbot.append(("完成了吗?", res))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
|
||||
@@ -109,9 +109,9 @@ def 解析源代码新(file_manifest, project_folder, llm_kwargs, plugin_kwargs,
|
||||
def 解析项目本身(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port):
|
||||
history = [] # 清空历史,以免输入溢出
|
||||
import glob
|
||||
file_manifest = [f for f in glob.glob('./*.py') if ('test_project' not in f) and ('gpt_log' not in f)] + \
|
||||
[f for f in glob.glob('./crazy_functions/*.py') if ('test_project' not in f) and ('gpt_log' not in f)]+ \
|
||||
[f for f in glob.glob('./request_llm/*.py') if ('test_project' not in f) and ('gpt_log' not in f)]
|
||||
file_manifest = [f for f in glob.glob('./*.py')] + \
|
||||
[f for f in glob.glob('./crazy_functions/**/*.py')]+ \
|
||||
[f for f in glob.glob('./request_llm/**/*.py')]
|
||||
project_folder = './'
|
||||
if len(file_manifest) == 0:
|
||||
report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何python文件: {txt}")
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from toolbox import update_ui
|
||||
from toolbox import CatchException, report_execption, write_results_to_file
|
||||
from toolbox import CatchException, report_execption
|
||||
from toolbox import write_history_to_file, promote_file_to_downloadzone
|
||||
from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||
fast_debug = False
|
||||
|
||||
|
||||
def 解析Paper(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt):
|
||||
@@ -17,32 +17,29 @@ def 解析Paper(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbo
|
||||
chatbot.append((i_say_show_user, "[Local Message] waiting gpt response."))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
if not fast_debug:
|
||||
msg = '正常'
|
||||
# ** gpt request **
|
||||
gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say_show_user, llm_kwargs, chatbot, history=[], sys_prompt=system_prompt) # 带超时倒计时
|
||||
|
||||
chatbot[-1] = (i_say_show_user, gpt_say)
|
||||
history.append(i_say_show_user); history.append(gpt_say)
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面
|
||||
if not fast_debug: time.sleep(2)
|
||||
msg = '正常'
|
||||
gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say_show_user, llm_kwargs, chatbot, history=[], sys_prompt=system_prompt) # 带超时倒计时
|
||||
chatbot[-1] = (i_say_show_user, gpt_say)
|
||||
history.append(i_say_show_user); history.append(gpt_say)
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面
|
||||
time.sleep(2)
|
||||
|
||||
all_file = ', '.join([os.path.relpath(fp, project_folder) for index, fp in enumerate(file_manifest)])
|
||||
i_say = f'根据以上你自己的分析,对全文进行概括,用学术性语言写一段中文摘要,然后再写一段英文摘要(包括{all_file})。'
|
||||
chatbot.append((i_say, "[Local Message] waiting gpt response."))
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
if not fast_debug:
|
||||
msg = '正常'
|
||||
# ** gpt request **
|
||||
gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say, llm_kwargs, chatbot, history=history, sys_prompt=system_prompt) # 带超时倒计时
|
||||
msg = '正常'
|
||||
# ** gpt request **
|
||||
gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(i_say, i_say, llm_kwargs, chatbot, history=history, sys_prompt=system_prompt) # 带超时倒计时
|
||||
|
||||
chatbot[-1] = (i_say, gpt_say)
|
||||
history.append(i_say); history.append(gpt_say)
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面
|
||||
res = write_results_to_file(history)
|
||||
chatbot.append(("完成了吗?", res))
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面
|
||||
chatbot[-1] = (i_say, gpt_say)
|
||||
history.append(i_say); history.append(gpt_say)
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面
|
||||
res = write_history_to_file(history)
|
||||
promote_file_to_downloadzone(res, chatbot=chatbot)
|
||||
chatbot.append(("完成了吗?", res))
|
||||
yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
# @Time : 2023/4/19
|
||||
# @Author : Spike
|
||||
# @Descr :
|
||||
from toolbox import update_ui
|
||||
from toolbox import CatchException, report_execption, write_results_to_file, get_log_folder
|
||||
from toolbox import update_ui, get_conf
|
||||
from toolbox import CatchException
|
||||
from crazy_functions.crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
|
||||
|
||||
|
||||
@@ -30,14 +30,13 @@ def 猜你想问(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt
|
||||
|
||||
@CatchException
|
||||
def 清除缓存(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port):
|
||||
chatbot.append(['清除本地缓存数据', '执行中. 删除 gpt_log & private_upload'])
|
||||
chatbot.append(['清除本地缓存数据', '执行中. 删除数据'])
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
|
||||
import shutil, os
|
||||
gpt_log_dir = os.path.join(os.path.dirname(__file__), '..', 'gpt_log')
|
||||
private_upload_dir = os.path.join(os.path.dirname(__file__), '..', 'private_upload')
|
||||
shutil.rmtree(gpt_log_dir, ignore_errors=True)
|
||||
shutil.rmtree(private_upload_dir, ignore_errors=True)
|
||||
PATH_PRIVATE_UPLOAD, PATH_LOGGING = get_conf('PATH_PRIVATE_UPLOAD', 'PATH_LOGGING')
|
||||
shutil.rmtree(PATH_LOGGING, ignore_errors=True)
|
||||
shutil.rmtree(PATH_PRIVATE_UPLOAD, ignore_errors=True)
|
||||
|
||||
chatbot.append(['清除本地缓存数据', '执行完成'])
|
||||
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
||||
在新工单中引用
屏蔽一个用户