latex toolchain

crazy_functions/latex_utils.py

@@ -0,0 +1,175 @@
+from toolbox import update_ui, trimmed_format_exc
+from toolbox import CatchException, report_execption, write_results_to_file, zip_folder
+import os
+import re
+
+
+
+def merge_tex_files(project_foler, main_file):
+    # Get the directory of the main tex file
+
+    # re.findall(r"\\input\{(.*?)\}", main_file, re.M)
+    for s in reversed([q for q in re.finditer(r"\\input\{(.*?)\}", main_file, re.M)]):
+        f = s.group(1)
+        fp = os.path.join(project_foler, f)
+        with open(fp, 'r', encoding='utf-8', errors='replace') as fx:
+            c = fx.read()
+        c = merge_tex_files(project_foler, c)
+        main_file = main_file[:s.span()[0]] + c + main_file[s.span()[1]:]
+
+    return main_file
+
+class LinkTable():
+    def __init__(self, string, preserve=True) -> None:
+        self.string = string
+        self.preserve = preserve
+        self.next = None
+
+class LatexPaperSplit():
+    def __init__(self) -> None:
+        self.root = None
+
+    def merge_result(self, arr):
+        def remove_special_chars(s):
+            s.replace('%', 'Percent')
+            return s
+        
+        result_string = ""
+        
+        node = self.root
+        p = 0
+        while True:
+            if node.preserve:
+                result_string += node.string
+            else:
+                result_string += remove_special_chars(arr[p])
+                p += 1
+            node = node.next
+            if node is None: break
+
+        return result_string
+
+    def split(self, txt):
+        # def replace_with_hash()
+        root = LinkTable(txt, False)
+        def split_worker(root, pattern, flags=0):
+            lt = root
+            cnt = 0
+            while True:
+                if not lt.preserve:
+                    while True:
+                        res = re.search(pattern, lt.string, flags)
+                        if not res: break
+                        before = res.string[:res.span()[0]]
+                        this = res.group(0)
+                        # core = res.group(1)
+                        after = res.string[res.span()[1]:]
+                        
+                        lt.string = before
+                        tmp  = lt.next
+                        # ======
+                        if after.startswith('\n'):
+                            # move \n
+                            this = this + '\n'
+                            after = after[1:]
+                        mid = LinkTable(this, True)
+                        lt.next = mid
+                        # ======
+                        aft = LinkTable(after, False)
+                        mid.next = aft
+                        aft.next = tmp
+                        # ======
+                        lt = aft
+                lt = lt.next
+                cnt += 1
+                print(cnt)
+                if lt is None: break
+
+        # root 是链表的头
+        print('正在分解Latex源文件')
+        split_worker(root, r"(.*?)\\maketitle", re.DOTALL)
+        split_worker(root, r"\\section\{(.*?)\}")
+        split_worker(root, r"\\subsection\{(.*?)\}")
+        split_worker(root, r"\\subsubsection\{(.*?)\}")
+        split_worker(root, r"\\begin\{figure\}(.*?)\\end\{figure\}", re.DOTALL)
+        split_worker(root, r"\\begin\{figure\*\}(.*?)\\end\{figure\*\}", re.DOTALL)
+        split_worker(root, r"\\begin\{table\}(.*?)\\end\{table\}", re.DOTALL)
+        split_worker(root, r"\\begin\{table\*\}(.*?)\\end\{table\*\}", re.DOTALL)
+        split_worker(root, r"\\item ")
+        split_worker(root, r"\\begin\{(.*?)\}")
+        split_worker(root, r"\\end\{(.*?)\}")
+
+        res = []
+        node = root
+        while True:
+            res.append((node.string, node.preserve))
+            if len(node.string.strip('\n').strip(''))==0: node.preserve = True
+            if len(node.string.strip('\n').strip(''))<50: node.preserve = True
+            node = node.next
+            if node is None: break
+
+        print('======================================')
+        res_to_t = []
+        node = root
+        while True:
+            if not node.preserve:
+                print(node.string)
+                res_to_t.append(node.string)
+            print('======================================')
+            node = node.next
+            if node is None: break
+        print('======================================')
+        self.root = root
+        self.sp = res_to_t
+        return self.sp
+
+class LatexPaperFileGroup():
+    def __init__(self):
+        self.file_paths = []
+        self.file_contents = []
+        self.sp_file_contents = []
+        self.sp_file_index = []
+        self.sp_file_tag = []
+
+        # count_token
+        from request_llm.bridge_all import model_info
+        enc = model_info["gpt-3.5-turbo"]['tokenizer']
+        def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
+        self.get_token_num = get_token_num
+
+    def run_file_split(self, max_token_limit=1900):
+        """
+        将长文本分离开来
+        """
+        for index, file_content in enumerate(self.file_contents):
+            if self.get_token_num(file_content) < max_token_limit:
+                self.sp_file_contents.append(file_content)
+                self.sp_file_index.append(index)
+                self.sp_file_tag.append(self.file_paths[index])
+            else:
+                from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf
+                segments = breakdown_txt_to_satisfy_token_limit_for_pdf(file_content, self.get_token_num, max_token_limit)
+                for j, segment in enumerate(segments):
+                    self.sp_file_contents.append(segment)
+                    self.sp_file_index.append(index)
+                    self.sp_file_tag.append(self.file_paths[index] + f".part-{j}.tex")
+        print('Segmentation: done')
+
+    def merge_result(self):
+        self.file_result = ["" for _ in range(len(self.file_paths))]
+        for r, k in zip(self.sp_file_result, self.sp_file_index):
+            self.file_result[k] += r
+
+    def write_result(self):
+        manifest = []
+        for path, res in zip(self.file_paths, self.file_result):
+            with open(path + '.polish.tex', 'w', encoding='utf8') as f:
+                manifest.append(path + '.polish.tex')
+                f.write(res)
+        return manifest
+    
+    def zip_result(self):
+        import os, time
+        folder = os.path.dirname(self.file_paths[0])
+        t = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
+        zip_folder(folder, './gpt_log/', f'{t}-polished.zip')