import os, sys
import json

# 获取当前路径， 通过anchor文件获取项目root路径
this_file_path = os.path.split(os.path.realpath(__file__))[0]
this_path = this_file_path
root_path = this_file_path
while this_path:
    if os.path.exists(os.path.join(this_path, 'sosweety_root_anchor.py')):
        root_path = this_path
        break
    par_path = os.path.dirname(this_path)
    # print(par_path)
    if par_path == this_path:
        break
    else:
        this_path = par_path
sys.path.append(root_path)

from modules.sParser.sParser import sParser
from modules.knowledgebase.kb import KnowledgeBase

train_dir = 'data/train_zh_wiki'
train_dir = os.path.join(root_path, train_dir)
if not os.path.exists(train_dir):
    os.makedirs(train_dir)

# 解析出parse result file
parse_result_dir = 'parse_result'
parse_result_dir = os.path.join(train_dir, parse_result_dir)
if not os.path.exists(parse_result_dir):
    os.makedirs(parse_result_dir)

pos_tags_file_name = 'pos_tags_file'
pos_tags_file_path = os.path.join(parse_result_dir, pos_tags_file_name)

KB = KnowledgeBase()
parser = sParser(KB)

with open(pos_tags_file_path, 'w') as pos_tags_file:
    # 打开语料文件
    file_path = 'data/corpus/zh_wiki/wiki_test'
    file_path = os.path.join(root_path, file_path)
    file = open(file_path)
    line = file.readline()
    count = 0
    while line:
        count += 1
        if count % 5000 == 0:
            print('parsed %s sentence' % count)
        text = line.strip()

        try:
            ss_pos_tags = parser.text2ss_pos_tags(text)
            for pos_tags in ss_pos_tags:
                pos_tags_file.write(json.dumps(pos_tags, ensure_ascii=False) + '\n')
        except Exception:
            print('line %s decode error' % count)

        line = file.readline()
    file.close()

[((18, 0, 18, 26), 'sys.path.append', 'sys.path.append', ({(18, 16, 18, 25): 'root_path'}, {}), '(root_path)', False, 'import os, sys\n'), ((24, 12, 24, 46), 'os.path.join', 'os.path.join', ({(24, 25, 24, 34): 'root_path', (24, 36, 24, 45): 'train_dir'}, {}), '(root_path, train_dir)', False, 'import os, sys\n'), ((30, 19, 30, 60), 'os.path.join', 'os.path.join', ({(30, 32, 30, 41): 'train_dir', (30, 43, 30, 59): 'parse_result_dir'}, {}), '(train_dir, parse_result_dir)', False, 'import os, sys\n'), ((35, 21, 35, 71), 'os.path.join', 'os.path.join', ({(35, 34, 35, 50): 'parse_result_dir', (35, 52, 35, 70): 'pos_tags_file_name'}, {}), '(parse_result_dir, pos_tags_file_name)', False, 'import os, sys\n'), ((37, 5, 37, 20), 'modules.knowledgebase.kb.KnowledgeBase', 'KnowledgeBase', ({}, {}), '()', False, 'from modules.knowledgebase.kb import KnowledgeBase\n'), ((38, 9, 38, 20), 'modules.sParser.sParser.sParser', 'sParser', ({(38, 17, 38, 19): 'KB'}, {}), '(KB)', False, 'from modules.sParser.sParser import sParser\n'), ((12, 15, 12, 41), 'os.path.dirname', 'os.path.dirname', ({(12, 31, 12, 40): 'this_path'}, {}), '(this_path)', False, 'import os, sys\n'), ((25, 7, 25, 32), 'os.path.exists', 'os.path.exists', ({(25, 22, 25, 31): 'train_dir'}, {}), '(train_dir)', False, 'import os, sys\n'), ((26, 4, 26, 26), 'os.makedirs', 'os.makedirs', ({(26, 16, 26, 25): 'train_dir'}, {}), '(train_dir)', False, 'import os, sys\n'), ((31, 7, 31, 39), 'os.path.exists', 'os.path.exists', ({(31, 22, 31, 38): 'parse_result_dir'}, {}), '(parse_result_dir)', False, 'import os, sys\n'), ((32, 4, 32, 33), 'os.makedirs', 'os.makedirs', ({(32, 16, 32, 32): 'parse_result_dir'}, {}), '(parse_result_dir)', False, 'import os, sys\n'), ((43, 16, 43, 50), 'os.path.join', 'os.path.join', ({(43, 29, 43, 38): 'root_path', (43, 40, 43, 49): 'file_path'}, {}), '(root_path, file_path)', False, 'import os, sys\n'), ((5, 31, 5, 57), 'os.path.realpath', 'os.path.realpath', ({(5, 48, 5, 56): '__file__'}, {}), '(__file__)', False, 'import os, sys\n'), ((9, 22, 9, 72), 'os.path.join', 'os.path.join', ({(9, 35, 9, 44): 'this_path', (9, 46, 9, 71): '"""sosweety_root_anchor.py"""'}, {}), "(this_path, 'sosweety_root_anchor.py')", False, 'import os, sys\n'), ((56, 36, 56, 76), 'json.dumps', 'json.dumps', (), '', False, 'import json\n')]