text: #
  # SPDX-FileCopyrightText: 2017 Facebook, Inc.
  #
  # SPDX-License-Identifier: BSD-3-Clause
  #
  # Copyright (c) 2017-present, Facebook, Inc.
  # All rights reserved.
  #
  # This source code is licensed under the license found in the
  # LICENSE file in the root directory of this source tree.
  #
  
  '''
  probing tasks
  '''
  
  from __future__ import absolute_import, division, unicode_literals
  
  import os
  import io
  import copy
  import logging
  import numpy as np
  
  from senteval.tools.validation import SplitClassifier
  
  
  class PROBINGEval(object):
      def __init__(self, task, task_path, seed=1111):
          self.seed = seed
          self.task = task
          logging.debug('***** (Probing) Transfer task : %s classification *****', self.task.upper())
          self.task_data = {'train': {'X': [], 'y': []},
                            'dev': {'X': [], 'y': []},
                            'test': {'X': [], 'y': []}}
          self.loadFile(task_path)
          logging.info('Loaded %s train - %s dev - %s test for %s' %
                       (len(self.task_data['train']['y']), len(self.task_data['dev']['y']),
                        len(self.task_data['test']['y']), self.task))
  
      def do_prepare(self, params, prepare):
          samples = self.task_data['train']['X'] + self.task_data['dev']['X'] + \
                    self.task_data['test']['X']
          return prepare(params, samples)
  
      def loadFile(self, fpath):
          self.tok2split = {'tr': 'train', 'va': 'dev', 'te': 'test'}
          with io.open(fpath, 'r', encoding='utf-8') as f:
              for line in f:
                  line = line.rstrip().split('\t')
                  self.task_data[self.tok2split[line[0]]]['X'].append(line[-1].split())
                  self.task_data[self.tok2split[line[0]]]['y'].append(line[1])
  
          labels = sorted(np.unique(self.task_data['train']['y']))
          self.tok2label = dict(zip(labels, range(len(labels))))
          self.nclasses = len(self.tok2label)
  
          for split in self.task_data:
              for i, y in enumerate(self.task_data[split]['y']):
                  self.task_data[split]['y'][i] = self.tok2label[y]
  
      def run(self, params, batcher):
          task_embed = {'train': {}, 'dev': {}, 'test': {}}
          bsize = params.batch_size
          logging.info('Computing embeddings for train/dev/test')
          for key in self.task_data:
              # Sort to reduce padding
              sorted_data = sorted(zip(self.task_data[key]['X'],
                                       self.task_data[key]['y']),
                                   key=lambda z: (len(z[0]), z[1]))
              self.task_data[key]['X'], self.task_data[key]['y'] = map(list, zip(*sorted_data))
  
              task_embed[key]['X'] = []
              for ii in range(0, len(self.task_data[key]['y']), bsize):
                  batch = self.task_data[key]['X'][ii:ii + bsize]
                  embeddings = batcher(params, batch)
                  task_embed[key]['X'].append(embeddings)
              task_embed[key]['X'] = np.vstack(task_embed[key]['X'])
              task_embed[key]['y'] = np.array(self.task_data[key]['y'])
          logging.info('Computed embeddings')
  
          config_classifier = {'nclasses': self.nclasses, 'seed': self.seed,
                               'usepytorch': params.usepytorch,
                               'classifier': params.classifier}
  
          if self.task == "WordContent" and params.classifier['nhid'] > 0:
              config_classifier = copy.deepcopy(config_classifier)
              config_classifier['classifier']['nhid'] = 0
              print(params.classifier['nhid'])
  
          clf = SplitClassifier(X={'train': task_embed['train']['X'],
                                   'valid': task_embed['dev']['X'],
                                   'test': task_embed['test']['X']},
                                y={'train': task_embed['train']['y'],
                                   'valid': task_embed['dev']['y'],
                                   'test': task_embed['test']['y']},
                                config=config_classifier)
  
          devacc, testacc = clf.run()
          logging.debug('\nDev acc : %.1f Test acc : %.1f for %s classification\n' % (devacc, testacc, self.task.upper()))
  
          return {'devacc': devacc, 'acc': testacc,
                  'ndev': len(task_embed['dev']['X']),
                  'ntest': len(task_embed['test']['X'])}
  
  """
  Surface Information
  """
  class LengthEval(PROBINGEval):
      def __init__(self, task_path, seed=1111):
          task_path = os.path.join(task_path, 'sentence_length.txt')
          # labels: bins
          PROBINGEval.__init__(self, 'Length', task_path, seed)
  
  class WordContentEval(PROBINGEval):
      def __init__(self, task_path, seed=1111):
          task_path = os.path.join(task_path, 'word_content.txt')
          # labels: 200 target words
          PROBINGEval.__init__(self, 'WordContent', task_path, seed)
  
  """
  Latent Structural Information
  """
  class DepthEval(PROBINGEval):
      def __init__(self, task_path, seed=1111):
          task_path = os.path.join(task_path, 'tree_depth.txt')
          # labels: bins
          PROBINGEval.__init__(self, 'Depth', task_path, seed)
  
  class TopConstituentsEval(PROBINGEval):
      def __init__(self, task_path, seed=1111):
          task_path = os.path.join(task_path, 'top_constituents.txt')
          # labels: 'PP_NP_VP_.' .. (20 classes)
          PROBINGEval.__init__(self, 'TopConstituents', task_path, seed)
  
  class BigramShiftEval(PROBINGEval):
      def __init__(self, task_path, seed=1111):
          task_path = os.path.join(task_path, 'bigram_shift.txt')
          # labels: 0 or 1
          PROBINGEval.__init__(self, 'BigramShift', task_path, seed)
  
  # TODO: Voice?
  
  """
  Latent Semantic Information
  """
  
  class TenseEval(PROBINGEval):
      def __init__(self, task_path, seed=1111):
          task_path = os.path.join(task_path, 'past_present.txt')
          # labels: 'PRES', 'PAST'
          PROBINGEval.__init__(self, 'Tense', task_path, seed)
  
  class SubjNumberEval(PROBINGEval):
      def __init__(self, task_path, seed=1111):
          task_path = os.path.join(task_path, 'subj_number.txt')
          # labels: 'NN', 'NNS'
          PROBINGEval.__init__(self, 'SubjNumber', task_path, seed)
  
  class ObjNumberEval(PROBINGEval):
      def __init__(self, task_path, seed=1111):
          task_path = os.path.join(task_path, 'obj_number.txt')
          # labels: 'NN', 'NNS'
          PROBINGEval.__init__(self, 'ObjNumber', task_path, seed)
  
  class OddManOutEval(PROBINGEval):
      def __init__(self, task_path, seed=1111):
          task_path = os.path.join(task_path, 'odd_man_out.txt')
          # labels: 'O', 'C'
          PROBINGEval.__init__(self, 'OddManOut', task_path, seed)
  
  class CoordinationInversionEval(PROBINGEval):
      def __init__(self, task_path, seed=1111):
          task_path = os.path.join(task_path, 'coordination_inversion.txt')
          # labels: 'O', 'I'
          PROBINGEval.__init__(self, 'CoordinationInversion', task_path, seed)
  