增加word2vec训练

.idea/workspace.xml

@@ -20,6 +20,7 @@
   </component>
   <component name="ChangeListManager">
     <list default="true" id="818087ba-af84-4167-ab4f-8ca76742a4b1" name="Default" comment="">
+      <change afterPath="$PROJECT_DIR$/image/词库构建.png" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
     </list>
@@ -51,9 +52,14 @@
   </component>
   <component name="PropertiesComponent">
     <property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
-    <property name="last_opened_file_path" value="$PROJECT_DIR$" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$/image" />
     <property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
   </component>
+  <component name="RecentsManager">
+    <key name="CopyFile.RECENT_KEYS">
+      <recent name="E:\work\Medical-nlp\image" />
+    </key>
+  </component>
   <component name="RunManager" selected="Python.zhiwang_dict">
     <configuration name="baidu" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
       <module name="Medical-nlp" />
@@ -164,22 +170,22 @@
     </option>
   </component>
   <component name="WindowStateProjectService">
-    <state width="1515" height="273" key="GridCell.Tab.0.bottom" timestamp="1602082310445">
+    <state width="1515" height="273" key="GridCell.Tab.0.bottom" timestamp="1602083248084">
       <screen x="0" y="0" width="1536" height="824" />
     </state>
-    <state width="1515" height="273" key="GridCell.Tab.0.bottom/0.0.1536.824@0.0.1536.824" timestamp="1602082310445" />
-    <state width="1515" height="273" key="GridCell.Tab.0.center" timestamp="1602082310444">
+    <state width="1515" height="273" key="GridCell.Tab.0.bottom/0.0.1536.824@0.0.1536.824" timestamp="1602083248084" />
+    <state width="1515" height="273" key="GridCell.Tab.0.center" timestamp="1602083248084">
       <screen x="0" y="0" width="1536" height="824" />
     </state>
-    <state width="1515" height="273" key="GridCell.Tab.0.center/0.0.1536.824@0.0.1536.824" timestamp="1602082310444" />
-    <state width="1515" height="273" key="GridCell.Tab.0.left" timestamp="1602082310444">
+    <state width="1515" height="273" key="GridCell.Tab.0.center/0.0.1536.824@0.0.1536.824" timestamp="1602083248084" />
+    <state width="1515" height="273" key="GridCell.Tab.0.left" timestamp="1602083248084">
       <screen x="0" y="0" width="1536" height="824" />
     </state>
-    <state width="1515" height="273" key="GridCell.Tab.0.left/0.0.1536.824@0.0.1536.824" timestamp="1602082310444" />
-    <state width="1515" height="273" key="GridCell.Tab.0.right" timestamp="1602082310444">
+    <state width="1515" height="273" key="GridCell.Tab.0.left/0.0.1536.824@0.0.1536.824" timestamp="1602083248084" />
+    <state width="1515" height="273" key="GridCell.Tab.0.right" timestamp="1602083248084">
       <screen x="0" y="0" width="1536" height="824" />
     </state>
-    <state width="1515" height="273" key="GridCell.Tab.0.right/0.0.1536.824@0.0.1536.824" timestamp="1602082310444" />
+    <state width="1515" height="273" key="GridCell.Tab.0.right/0.0.1536.824@0.0.1536.824" timestamp="1602083248084" />
     <state x="431" y="145" width="672" height="678" key="search.everywhere.popup" timestamp="1599057981367">
       <screen x="0" y="0" width="1536" height="824" />
     </state>

src/medical_word2vec.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import jieba
+import warnings
+import logging
+import os.path
+import sys
+import multiprocessing
+
+from gensim.models import Word2Vec
+from gensim.models.word2vec import LineSentence
+filePath = 'corpus_1.txt'
+fileSegWordDonePath = 'corpusSegDone_1.txt'
+warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')
+
+
+# 打印中文列表
+def PrintListChinese(list):
+    for i in range(len(list)):
+        print(list[i])
+
+
+fileTrainRead = []
+with open(filePath, 'r') as fileTrainRaw:
+    for line in fileTrainRaw:  # 按行读取文件
+        fileTrainRead.append(line)
+
+# jieba分词后保存在列表中
+fileTrainSeg = []
+for i in range(len(fileTrainRead)):
+    fileTrainSeg.append([' '.join(list(jieba.cut(fileTrainRead[i][9:-11], cut_all=False)))])
+    if i % 100 == 0:
+        print(i)
+
+# 保存分词结果到文件中
+with open(fileSegWordDonePath, 'w', encoding='utf-8') as fW:
+    for i in range(len(fileTrainSeg)):
+        fW.write(fileTrainSeg[i][0])
+        fW.write('\n')
+
+"""
+gensim word2vec获取词向量
+"""
+
+if __name__ == '__main__':
+    program = os.path.basename(sys.argv[0])  # 读取当前文件的文件名
+    logger = logging.getLogger(program)
+    logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s', level=logging.INFO)
+    logger.info("running %s" % ' '.join(sys.argv))
+
+    # inp为输入语料, outp1为输出模型, outp2为vector格式的模型
+    inp = 'corpusSegDone_1.txt'
+    out_model = 'corpusSegDone_1.model'
+    out_vector = 'corpusSegDone_1.vector'
+
+    # 训练skip-gram模型
+    model = Word2Vec(LineSentence(inp), size=50, window=5, min_count=5,
+                     workers=multiprocessing.cpu_count())
+
+    # 保存模型
+    model.save(out_model)
+    # 保存词向量
+    model.wv.save_word2vec_format(out_vector, binary=False)