





import argparse
import glob
import logging
import os
from argparse import Namespace
from importlib import import_module

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from seqeval.metrics import accuracy_score, fa_score, precision_score, recall_score
from torch.nn import CrossEntropyLoss
from torch.utils.data import DataLoader, TensorDataset
from utils_ner import TokenClassificationTask


__A     =    logging.getLogger(__name__)



class        _SCREAMING_SNAKE_CASE						(     __lowercase   ):







 '''simple docstring'''
 lowercase_		=      '''token-classification'''

 def __init__(self    :       str	,			UpperCAmelCase_    :       List[str])	->Dict:



   '''simple docstring'''



   if type(UpperCAmelCase__) == dict:
     lowerCamelCase__:				Optional[Any]        =Namespace(**UpperCAmelCase__)
   lowerCamelCase__:				str        =import_module("tasks")
   try:
     lowerCamelCase__:				List[str]        =getattr(UpperCAmelCase__	,			hparams.task_type)
     lowerCamelCase__:				Any        =token_classification_task_clazz()
   except AttributeError:
     raise ValueError(
         F"""Task {hparams.task_type} needs to be defined as a TokenClassificationTask subclass in {module}. """
         F"""Available tasks classes are: {TokenClassificationTask.__subclasses__()}""")
   lowerCamelCase__:				Dict        =self.token_classification_task.get_labels(hparams.labels)
   lowerCamelCase__:				str        =CrossEntropyLoss().ignore_index
   super().__init__(UpperCAmelCase__	,			len(self.labels)	,			self.mode)

 def 							SCREAMING_SNAKE_CASE_ (self    :       int	,			**UpperCAmelCase_    :       int)	->List[str]:



   '''simple docstring'''



   return self.model(**UpperCAmelCase__)

 def 							SCREAMING_SNAKE_CASE_ (self    :       int	,			UpperCAmelCase_    :       Dict	,			UpperCAmelCase_    :       int)	->List[Any]:



   '''simple docstring'''



   lowerCamelCase__:				Dict        ={"input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3]}
   if self.config.model_type != "distilbert":
     lowerCamelCase__:				int        =(
         batch[2] if self.config.model_type in ["bert", "xlnet"] else None
     )  # XLM and RoBERTa don"t use token_type_ids

   lowerCamelCase__:				Optional[Any]        =self(**UpperCAmelCase__)
   lowerCamelCase__:				Any        =outputs[0]
   # tensorboard_logs = {"loss": loss, "rate": self.lr_scheduler.get_last_lr()[-1]}
   return {"loss": loss}

 def 							SCREAMING_SNAKE_CASE_ (self    :       Any)	->Any:



   '''simple docstring'''



   lowerCamelCase__:				List[str]        =self.hparams
   for mode in ["train", "dev", "test"]:
     lowerCamelCase__:				Optional[int]        =self._feature_file(UpperCAmelCase__)
     if os.path.exists(UpperCAmelCase__) and not args.overwrite_cache:
       logger.info("Loading features from cached file %s"	,			UpperCAmelCase__)
       lowerCamelCase__:				Tuple        =torch.load(UpperCAmelCase__)
     else:
       logger.info("Creating features from dataset file at %s"	,			args.data_dir)
       lowerCamelCase__:				Union[str, Any]        =self.token_classification_task.read_examples_from_file(args.data_dir	,			UpperCAmelCase__)
       lowerCamelCase__:				str        =self.token_classification_task.convert_examples_to_features(
           UpperCAmelCase__	,			self.labels	,			args.max_seq_length	,			self.tokenizer	,			cls_token_at_end=bool(self.config.model_type in ["xlnet"])	,			cls_token=self.tokenizer.cls_token	,			cls_token_segment_id=2 if self.config.model_type in ["xlnet"] else 0	,			sep_token=self.tokenizer.sep_token	,			sep_token_extra=UpperCAmelCase__	,			pad_on_left=bool(self.config.model_type in ["xlnet"])	,			pad_token=self.tokenizer.pad_token_id	,			pad_token_segment_id=self.tokenizer.pad_token_type_id	,			pad_token_label_id=self.pad_token_label_id	,			)
       logger.info("Saving features into cached file %s"	,			UpperCAmelCase__)
       torch.save(UpperCAmelCase__	,			UpperCAmelCase__)

 def 							SCREAMING_SNAKE_CASE_ (self    :       str	,			UpperCAmelCase_    :       int	,			UpperCAmelCase_    :       int	,			UpperCAmelCase_    :       bool = False)	->DataLoader:



   '''simple docstring'''



   lowerCamelCase__:				int        =self._feature_file(UpperCAmelCase__)
   logger.info("Loading features from cached file %s"	,			UpperCAmelCase__)
   lowerCamelCase__:				str        =torch.load(UpperCAmelCase__)
   lowerCamelCase__:				Union[str, Any]        =torch.tensor([f.input_ids for f in features]	,			dtype=torch.long)
   lowerCamelCase__:				Dict        =torch.tensor([f.attention_mask for f in features]	,			dtype=torch.long)
   if features[0].token_type_ids is not None:
     lowerCamelCase__:				int        =torch.tensor([f.token_type_ids for f in features]	,			dtype=torch.long)
   else:
     lowerCamelCase__:				Dict        =torch.tensor([0 for f in features]	,			dtype=torch.long)
     # HACK(we will not use this anymore soon)
   lowerCamelCase__:				Union[str, Any]        =torch.tensor([f.label_ids for f in features]	,			dtype=torch.long)
   return DataLoader(
       TensorDataset(UpperCAmelCase__	,			UpperCAmelCase__	,			UpperCAmelCase__	,			UpperCAmelCase__)	,			batch_size=UpperCAmelCase__)

 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[int]	,			UpperCAmelCase_    :       List[str]	,			UpperCAmelCase_    :       Union[str, Any])	->Dict:



   '''simple docstring'''



   """Compute validation""" ""
   lowerCamelCase__:				Optional[Any]        ={"input_ids": batch[0], "attention_mask": batch[1], "labels": batch[3]}
   if self.config.model_type != "distilbert":
     lowerCamelCase__:				Any        =(
         batch[2] if self.config.model_type in ["bert", "xlnet"] else None
     )  # XLM and RoBERTa don"t use token_type_ids
   lowerCamelCase__:				Tuple        =self(**UpperCAmelCase__)
   lowerCamelCase__  , lowerCamelCase__:				Union[str, Any]        =outputs[:2]
   lowerCamelCase__:				int        =logits.detach().cpu().numpy()
   lowerCamelCase__:				str        =inputs["labels"].detach().cpu().numpy()
   return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}

 def 							SCREAMING_SNAKE_CASE_ (self    :       int	,			UpperCAmelCase_    :       List[str])	->List[Any]:



   '''simple docstring'''



   lowerCamelCase__:				str        =torch.stack([x["val_loss"] for x in outputs]).mean()
   lowerCamelCase__:				Optional[Any]        =np.concatenate([x["pred"] for x in outputs]	,			axis=0)
   lowerCamelCase__:				int        =np.argmax(UpperCAmelCase__	,			axis=2)
   lowerCamelCase__:				Union[str, Any]        =np.concatenate([x["target"] for x in outputs]	,			axis=0)

   lowerCamelCase__:				int        =dict(enumerate(self.labels))
   lowerCamelCase__:				List[str]        =[[] for _ in range(out_label_ids.shape[0])]
   lowerCamelCase__:				int        =[[] for _ in range(out_label_ids.shape[0])]

   for i in range(out_label_ids.shape[0]):
     for j in range(out_label_ids.shape[1]):
       if out_label_ids[i, j] != self.pad_token_label_id:
         out_label_list[i].append(label_map[out_label_ids[i][j]])
         preds_list[i].append(label_map[preds[i][j]])

   lowerCamelCase__:				Union[str, Any]        ={
       "val_loss": val_loss_mean,
       "accuracy_score": accuracy_score(UpperCAmelCase__	,			UpperCAmelCase__),
       "precision": precision_score(UpperCAmelCase__	,			UpperCAmelCase__),
       "recall": recall_score(UpperCAmelCase__	,			UpperCAmelCase__),
       "f1": fa_score(UpperCAmelCase__	,			UpperCAmelCase__),
   }

   lowerCamelCase__:				Any        =dict(results.items())
   lowerCamelCase__:				Dict        =results
   return ret, preds_list, out_label_list

 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[Any]	,			UpperCAmelCase_    :       str)	->List[Any]:



   '''simple docstring'''



   lowerCamelCase__  , lowerCamelCase__  , lowerCamelCase__:				Dict        =self._eval_end(UpperCAmelCase__)
   lowerCamelCase__:				Dict        =ret["log"]
   return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}

 def 							SCREAMING_SNAKE_CASE_ (self    :       Any	,			UpperCAmelCase_    :       Optional[Any])	->Dict:



   '''simple docstring'''



   lowerCamelCase__  , lowerCamelCase__  , lowerCamelCase__:				int        =self._eval_end(UpperCAmelCase__)

   # Converting to the dict required by pl
   # https://github.com/PyTorchLightning/pytorch-lightning/blob/master/\
   # pytorch_lightning/trainer/logging.py#L139
   lowerCamelCase__:				List[Any]        =ret["log"]
   # `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
   return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}







 @staticmethod
 def 							SCREAMING_SNAKE_CASE_ (UpperCAmelCase_    :       int	,			UpperCAmelCase_    :       List[Any])	->int:



   '''simple docstring'''



   BaseTransformer.add_model_specific_args(UpperCAmelCase__	,			UpperCAmelCase__)
   parser.add_argument(
       "--task_type"	,			default="NER"	,			type=UpperCAmelCase__	,			help="Task type to fine tune in training (e.g. NER, POS, etc)")
   parser.add_argument(
       "--max_seq_length"	,			default=128	,			type=UpperCAmelCase__	,			help=(
           "The maximum total input sequence length after tokenization. Sequences longer "
           "than this will be truncated, sequences shorter will be padded."
       )	,			)

   parser.add_argument(
       "--labels"	,			default=""	,			type=UpperCAmelCase__	,			help="Path to a file containing all labels. If not specified, CoNLL-2003 labels are used."	,			)
   parser.add_argument(
       "--gpus"	,			default=0	,			type=UpperCAmelCase__	,			help="The number of GPUs allocated for this, it is by default 0 meaning none"	,			)

   parser.add_argument(
       "--overwrite_cache"	,			action="store_true"	,			help="Overwrite the cached training and evaluation sets")

   return parser


if __name__ == "__main__":
     __A     =    argparse.ArgumentParser()
     add_generic_args(parser, os.getcwd())
     __A     =    NERTransformer.add_model_specific_args(parser, os.getcwd())
     __A     =    parser.parse_args()
     __A     =    NERTransformer(args)
     __A     =    generic_train(model, args)

     if args.do_predict:
          # See https://github.com/huggingface/transformers/issues/3159
          # pl use this default format to create a checkpoint:
          # https://github.com/PyTorchLightning/pytorch-lightning/blob/master\
          # /pytorch_lightning/callbacks/model_checkpoint.py#L322
          __A     =    sorted(glob.glob(os.path.join(args.output_dir, "checkpoint-epoch=*.ckpt"), recursive=True))
          __A     =    model.load_from_checkpoint(checkpoints[-1])
          trainer.test(model)








import json
import os
import unittest

from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ,							unittest.TestCase   ):







 '''simple docstring'''
 lowercase_		=      CTRLTokenizer
 lowercase_		=      False
 lowercase_		=      False

 def 							SCREAMING_SNAKE_CASE_ (self    :       str)	->Union[str, Any]:



   '''simple docstring'''



   super().setUp()

   # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
   lowerCamelCase__:				Optional[Any]        =["adapt", "re@@", "a@@", "apt", "c@@", "t", "<unk>"]
   lowerCamelCase__:				Optional[Any]        =dict(zip(UpperCAmelCase_	,			range(len(UpperCAmelCase_))))
   lowerCamelCase__:				Union[str, Any]        =["#version: 0.2", "a p", "ap t</w>", "r e", "a d", "ad apt</w>", ""]
   lowerCamelCase__:				Union[str, Any]        ={"unk_token": "<unk>"}

   lowerCamelCase__:				Union[str, Any]        =os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES["vocab_file"])
   lowerCamelCase__:				Optional[Any]        =os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES["merges_file"])
   with open(self.vocab_file	,			"w"	,			encoding="utf-8") as fp:
     fp.write(json.dumps(UpperCAmelCase_) + "\n")
   with open(self.merges_file	,			"w"	,			encoding="utf-8") as fp:
     fp.write("\n".join(UpperCAmelCase_))

 def 							SCREAMING_SNAKE_CASE_ (self    :       List[str]	,			**UpperCAmelCase_    :       str)	->List[Any]:



   '''simple docstring'''



   kwargs.update(self.special_tokens_map)
   return CTRLTokenizer.from_pretrained(self.tmpdirname	,			**UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_ (self    :       List[Any]	,			UpperCAmelCase_    :       Optional[int])	->Optional[int]:



   '''simple docstring'''



   lowerCamelCase__:				int        ="adapt react readapt apt"
   lowerCamelCase__:				int        ="adapt react readapt apt"
   return input_text, output_text







 def 							SCREAMING_SNAKE_CASE_ (self    :       str)	->Union[str, Any]:



   '''simple docstring'''



   lowerCamelCase__:				List[str]        =CTRLTokenizer(self.vocab_file	,			self.merges_file	,			**self.special_tokens_map)
   lowerCamelCase__:				Union[str, Any]        ="adapt react readapt apt"
   lowerCamelCase__:				Tuple        ="adapt re@@ a@@ c@@ t re@@ adapt apt".split()
   lowerCamelCase__:				Dict        =tokenizer.tokenize(UpperCAmelCase_)
   self.assertListEqual(UpperCAmelCase_	,			UpperCAmelCase_)

   lowerCamelCase__:				Optional[int]        =tokens + [tokenizer.unk_token]

   lowerCamelCase__:				List[str]        =[0, 1, 2, 4, 5, 1, 0, 3, 6]
   self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_)	,			UpperCAmelCase_)

