text: # Copyright (c) 2020, Soohwan Kim. All rights reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  import os
  import re
  
  
  def bracket_filter(sentence, mode='phonetic'):
      new_sentence = str()
  
      if mode == 'phonetic':
          flag = False
  
          for ch in sentence:
              if ch == '(' and flag is False:
                  flag = True
                  continue
              if ch == '(' and flag is True:
                  flag = False
                  continue
              if ch != ')' and flag is False:
                  new_sentence += ch
  
      elif mode == 'spelling':
          flag = True
  
          for ch in sentence:
              if ch == '(':
                  continue
              if ch == ')':
                  if flag is True:
                      flag = False
                      continue
                  else:
                      flag = True
                      continue
              if ch != ')' and flag is True:
                  new_sentence += ch
  
      else:
          raise ValueError("Unsupported mode : {0}".format(mode))
  
      return new_sentence
  
  
  def special_filter(sentence, mode='phonetic', replace=None):
      SENTENCE_MARK = ['?', '!', '.']
      NOISE = ['o', 'n', 'u', 'b', 'l']
      EXCEPT = ['/', '+', '*', '-', '@', '$', '^', '&', '[', ']', '=', ':', ';', ',']
  
      new_sentence = str()
      for idx, ch in enumerate(sentence):
          if ch not in SENTENCE_MARK:
              if idx + 1 < len(sentence) and ch in NOISE and sentence[idx + 1] == '/':
                  continue
  
          if ch == '#':
              new_sentence += '샾'
  
          elif ch == '%':
              if mode == 'phonetic':
                  new_sentence += replace
              elif mode == 'spelling':
                  new_sentence += '%'
  
          elif ch not in EXCEPT:
              new_sentence += ch
  
      pattern = re.compile(r'\s\s+')
      new_sentence = re.sub(pattern, ' ', new_sentence.strip())
      return new_sentence
  
  
  def sentence_filter(raw_sentence, mode, replace=None):
      return special_filter(bracket_filter(raw_sentence, mode), mode, replace)
  
  
  def preprocess(dataset_path, mode='phonetic'):
      print('preprocess started..')
  
      audio_paths = list()
      transcripts = list()
  
      with open("train.txt") as f:
          for idx, line in enumerate(f.readlines()):
              audio_path, transcript = line.split('\t')
              transcript = transcript.replace('\n', '')
  
              audio_paths.append(audio_path)
              transcripts.append(transcript)
          print("성공")
  
      return audio_paths, transcripts
  
      # def preprocess(dataset_path, mode='phonetic'):
      # print('preprocess started..')
  
      # audio_paths = list()
      # transcripts = list()
  
      # percent_files = {
      #     '087797': '퍼센트',
      #     '215401': '퍼센트',
      #     '284574': '퍼센트',
      #     '397184': '퍼센트',
      #     '501006': '프로',
      #     '502173': '프로',
      #     '542363': '프로',
      #     '581483': '퍼센트'
      # }
  
      # # 이 부분 손대야함, 만들어놓은 메모로 하면 되겠다.
      # for folder in os.listdir(dataset_path):
      #     # folder : {KsponSpeech_01, ..., KsponSpeech_05}
      #     if not folder.startswith('KsponSpeech'):
      #         continue
      #     path = os.path.join(dataset_path, folder)
      #     for idx, subfolder in enumerate(os.listdir(path)):
      #         path = os.path.join(dataset_path, folder, subfolder)
  
      #         for jdx, file in enumerate(os.listdir(path)):
      #             if file.endswith('.txt'):
      #                 with open(os.path.join(path, file), "r", encoding='cp949') as f:
      #                     raw_sentence = f.read()
      #                     if file[12:18] in percent_files.keys():
      #                         new_sentence = sentence_filter(raw_sentence, mode, percent_files[file[12:18]])
      #                     else:
      #                         new_sentence = sentence_filter(raw_sentence, mode=mode)
  
      #                 audio_paths.append(os.path.join(folder, subfolder, file))
      #                 transcripts.append(new_sentence)
  
      #             else:
      #                 continue
  
      # return audio_paths, transcripts