text: # Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  """This file contains some public functions
  """
  
  
  def convert_tokenizer_res_to_old_version(tokenized_res):
      if isinstance(tokenized_res, list):
          return tokenized_res
      if isinstance(tokenized_res, dict):
          if len(tokenized_res['input_ids']) == 0 or not isinstance(
                  tokenized_res['input_ids'][0], list):
              return tokenized_res
          else:
              res = []
              for idx in range(len(tokenized_res['input_ids'])):
                  temp_dict = {}
                  key_list = list(tokenized_res.keys())
                  for key in key_list:
                      temp_dict[key] = tokenized_res[key][idx]
                  res.append(temp_dict)
              return res
      else:
          raise ValueError('unsupported result type')
  
  
  def cal_score(match_list, sorted_token):
      over_all = []
      miss = 0
      for i in match_list:
          over_all.extend(i[0])
  
      score_dic = {}
      for i in sorted_token:
          split_time = over_all.count(i[0])
          if split_time:
              score_dic[i[0]] = i[2] / split_time
          else:
              score_dic[i[0]] = 0.0
      if miss != 0:
          print(miss)
  
      score = []
      for i in range(len(match_list)):
          cur_score = 0.0
          for j in match_list[i][0]:
              if j == -1:
                  continue
              cur_score += score_dic[j]
          score.append([str(match_list[i][1]), match_list[i][2], cur_score])
      return score
  
  
  def match(context, context_seg, sorted_token):
      result = []
      pointer1 = 0  # point at the context
      pointer2 = 0  # point at the sorted_token array
      for i in range(len(context_seg)):
          seg_start_idx = context.find(context_seg[i], pointer1)
          if seg_start_idx < 0:
              print("Error: token not in context")
          seg_end_idx = seg_start_idx + len(context_seg[i])
  
          cur_set = []
          while pointer2 < len(sorted_token):
              while pointer2 < len(sorted_token) and sorted_token[pointer2][1][
                      1] <= seg_start_idx:
                  pointer2 += 1
              if pointer2 >= len(sorted_token):
                  break
              if sorted_token[pointer2][1][0] >= seg_end_idx:
                  break
              cur_set.append(sorted_token[pointer2][0])
              pointer2 += 1
          result.append([cur_set, i, context_seg[i]])
          pointer2 -= 1
          pointer1 = seg_end_idx
      score = cal_score(result, sorted_token)
      return score
  