#! /usr/bin/env python3

import re
import copy
from NLP21 import *
from NLP25 import *
from NLP26 import *
from NLP27 import *

def read_and_write_file_references4():
    lines = read_gz(filename)
    dictionaries = make_json_dictionaries(lines)
    basic_infos = get_parsed_basic_infos4(dictionaries)
    write_basic_infos(basic_infos)

def get_parsed_basic_infos4(dictionaries):
    ret = []
    for d in dictionaries:
        ret.append(get_parsed_basic_info4(d))
    return ret

def get_parsed_basic_info4(d):
    info = get_basic_info(d)
    dictionary = get_dictionary_of_basic_info4(info)
    ret_d = {}
    ret_d['title'] = d['title']
    ret_d['basic_info'] = dictionary
    return ret_d

def get_dictionary_of_basic_info4(info):
    if info is None:
        return None
    lines = info.rstrip().split('\n')
    d = {}
    content = lines[1 : -1]
    for l in content:
        key, value = parse_key_value4(l)
        if key is not None:
            d[key] = value
    return d

def parse_key_value4(line):
    key, value = parse_key_value(line)
    if key is None:
        return None, None
    else:
        key, value = remove_emphasis(key), remove_emphasis(value)
        key, value = remove_internal_links(key), remove_internal_links(value)
        return remove_some(key), remove_some(value)

def remove_some(what):
    dup = copy.deepcopy(what)
    p1 = re.compile('(<br>|<br/>|<br\s+/>)')
    p2 = re.compile('&nbsp;')
    p3 = re.compile('(\[\[|\]\])')
    p4 = re.compile('<!--.*?-->')
    p5 = re.compile('<ref.*?>.*?</ref>')
    p6 = re.compile('<refernces.*?/>')
    p7 = re.compile('<ref.*?/>')
    p8 = re.compile('File:')
    p9 = re.compile('ファイル:')
    p10 = re.compile('{{(l|L)ang\|(.*?)\|.*?}}')
    p11 = re.compile('<small>(.*?)</small>')
    p12 = re.compile('{{仮リンク\|.*?\|.*?\|(.*?)}}')
    p13 = re.compile('<sup>(.*?)</sup>')
    p14 = re.compile('{{Smaller\|(.*?)}}')
    p15 = re.compile('{{small\|(.*?)}}')
    p16 = re.compile('<span>.*</span>')
    p17 = re.compile('<b>(.*)</b>')
    p18 = re.compile('<small>')
    p19 = re.compile('<ref>.*')
    p20 = re.compile('<ref.*>')
    p21 = re.compile('{{.*')
    dup = remove_pattern_from3(dup, p1)
    dup = remove_pattern_from2(dup, p2)
    dup = remove_pattern_from2(dup, p3)
    dup = remove_pattern_from2(dup, p4)
    dup = remove_pattern_from2(dup, p5)
    dup = remove_pattern_from2(dup, p6)
    dup = remove_pattern_from2(dup, p7)
    dup = remove_pattern_from3(dup, p8)
    dup = remove_pattern_from3(dup, p9)
    dup = remove_pattern_from4(dup, p10)
    dup = remove_pattern_from(dup, p11)
    dup = remove_pattern_from(dup, p12)
    dup = remove_pattern_from(dup, p13)
    dup = remove_pattern_from(dup, p14)
    dup = remove_pattern_from(dup, p15)
    dup = remove_pattern_from2(dup, p16)
    dup = remove_pattern_from(dup, p17)
    dup = remove_pattern_from2(dup, p18)
    dup = remove_pattern_from2(dup, p19)
    dup = remove_pattern_from2(dup, p20)
    dup = remove_pattern_from2(dup, p21)
    return dup

def remove_pattern_from2(what, pattern):
    dup = copy.deepcopy(what)
    while pattern.search(dup) is not None:
        m = pattern.search(dup)
        dup = re.sub(pattern, r'', dup)
    return dup

def remove_pattern_from3(what, pattern):
    dup = copy.deepcopy(what)
    while pattern.search(dup) is not None:
        m = pattern.search(dup)
        dup = re.sub(pattern, r' ', dup)
    return dup

def remove_pattern_from4(what, pattern):
    dup = copy.deepcopy(what)
    while pattern.search(dup) is not None:
        m = pattern.search(dup)
        dup = re.sub(pattern, r'\2', dup)
    return dup

def remove_pattern_from5(what, pattern):
    dup = copy.deepcopy(what)
    while pattern.search(dup) is not None:
        m = pattern.search(dup)
        dup = re.sub(pattern, r'\3', dup)
    return dup

if __name__ == '__main__':
    read_and_write_file_references4()
