class_docstr: Text data processor that reads parallel source and target text.
  This can be used in, e.g., seq2seq models.
  
  Args:
      hparams (dict): Hyperparameters. See :meth:`default_hparams` for the
          defaults.
  
  By default, the processor reads raw data files, performs tokenization,
  batching and other pre-processing steps, and results in a TF Dataset
  whose element is a python `dict` including six fields:
  
      - "source_text":
          A string Tensor of shape `[batch_size, max_time]` containing
          the **raw** text toknes of source sequences. `max_time` is the
          length of the longest sequence in the batch.
          Short sequences in the batch are padded with **empty string**.
          By default only EOS token is appended to each sequence.
          Out-of-vocabulary tokens are **NOT** replaced with UNK.
      - "source_text_ids":
          An `int64` Tensor of shape `[batch_size, max_time]`
          containing the token indexes of source sequences.
      - "source_length":
          An `int` Tensor of shape `[batch_size]` containing the
          length of each source sequence in the batch (including BOS and/or
          EOS if added).
      - "target_text":
          A string Tensor as "source_text" but for target sequences. By
          default both BOS and EOS are added.
      - "target_text_ids":
          An `int64` Tensor as "source_text_ids" but for target sequences.
      - "target_length":
          An `int` Tensor of shape `[batch_size]` as "source_length" but for
          target sequences.
  
  If :attr:`'variable_utterance'` is set to `True` in :attr:`'source_dataset'`
  and/or :attr:`'target_dataset'` of :attr:`hparams`, the corresponding
  fields "source_*" and/or "target_*" are respectively changed to contain
  variable utterance text data, as in :class:`~texar.tf.data.MonoTextData`.
  
  The above field names can be accessed through :attr:`source_text_name`,
  :attr:`source_text_id_name`, :attr:`source_length_name`,
  :attr:`source_utterance_cnt_name`, and those prefixed with `target_`,
  respectively.
  
  Example:
  
      .. code-block:: python
  
          hparams={
              'source_dataset': {'files': 's', 'vocab_file': 'vs'},
              'target_dataset': {'files': ['t1', 't2'], 'vocab_file': 'vt'},
              'batch_size': 1
          }
          data = PairedTextData(hparams)
          iterator = DataIterator(data)
          batch = iterator.get_next()
  
          iterator.switch_to_dataset(sess) # initializes the dataset
          batch_ = sess.run(batch)
          # batch_ == {
          #    'source_text': [['source', 'sequence', '<EOS>']],
          #    'source_text_ids': [[5, 10, 2]],
          #    'source_length': [3]
          #    'target_text': [['<BOS>', 'target', 'sequence', '1', '<EOS>']],
          #    'target_text_ids': [[1, 6, 10, 20, 2]],
          #    'target_length': [5]
          # }
class_docstr_tok: ["Text","data","processor","that","reads","parallel","source","and","target","text",".","This","can","be","used","in",",","e.g.",",","seq2seq","models",".","Args",":","hparams","(","dict",")",":","Hyperparameters",".","See",":","meth",":","`","default_hparams","`","for","the","defaults",".","By","default",",","the","processor","reads","raw","data","files",",","performs","tokenization",",","batching","and","other","pre-processing","steps",",","and","results","in","a","TF","Dataset","whose","element","is","a","python","`","dict","`","including","six","fields",":","-","``","source_text","''",":","A","string","Tensor","of","shape","`","[","batch_size",",","max_time","]","`","containing","the","*","*","raw","*","*","text","toknes","of","source","sequences",".","`","max_time","`","is","the","length","of","the","longest","sequence","in","the","batch",".","Short","sequences","in","the","batch","are","padded","with","*","*","empty","string","*","*",".","By","default","only","EOS","token","is","appended","to","each","sequence",".","Out-of-vocabulary","tokens","are","*","*","NOT","*","*","replaced","with","UNK",".","-","``","source_text_ids","''",":","An","`","int64","`","Tensor","of","shape","`","[","batch_size",",","max_time","]","`","containing","the","token","indexes","of","source","sequences",".","-","``","source_length","''",":","An","`","int","`","Tensor","of","shape","`","[","batch_size","]","`","containing","the","length","of","each","source","sequence","in","the","batch","(","including","BOS","and\/or","EOS","if","added",")",".","-","``","target_text","''",":","A","string","Tensor","as","``","source_text","''","but","for","target","sequences",".","By","default","both","BOS","and","EOS","are","added",".","-","``","target_text_ids","''",":","An","`","int64","`","Tensor","as","``","source_text_ids","''","but","for","target","sequences",".","-","``","target_length","''",":","An","`","int","`","Tensor","of","shape","`","[","batch_size","]","`","as","``","source_length","''","but","for","target","sequences",".","If",":","attr",":","`","'variable_utterance","'","`","is","set","to","`","True","`","in",":","attr",":","`","'source_dataset","'","`","and\/or",":","attr",":","`","'target_dataset","'","`","of",":","attr",":","`","hparams","`",",","the","corresponding","fields","``","source_","*","''","and\/or","``","target_","*","''","are","respectively","changed","to","contain","variable","utterance","text","data",",","as","in",":","class",":","`","~texar.tf.data.MonoTextData","`",".","The","above","field","names","can","be","accessed","through",":","attr",":","`","source_text_name","`",",",":","attr",":","`","source_text_id_name","`",",",":","attr",":","`","source_length_name","`",",",":","attr",":","`","source_utterance_cnt_name","`",",","and","those","prefixed","with","`","target_","`",",","respectively",".","Example",":","..","code-block",":",":","python","hparams=","{","'source_dataset","'",":","{","'files","'",":","'s","'",",","'vocab_file","'",":","'vs","'","}",",","'target_dataset","'",":","{","'files","'",":","[","'t1","'",",","'t2","'","]",",","'vocab_file","'",":","'vt","'","}",",","'batch_size","'",":","1","}","data","=","PairedTextData","(","hparams",")","iterator","=","DataIterator","(","data",")","batch","=","iterator.get_next","(",")","iterator.switch_to_dataset","(","sess",")","#","initializes","the","dataset","batch_","=","sess.run","(","batch",")","#","batch_","==","{","#","'source_text","'",":","[","[","'source","'",",","'sequence","'",",","'","<","EOS",">","'","]","]",",","#","'source_text_ids","'",":","[","[","5",",","10",",","2","]","]",",","#","'source_length","'",":","[","3","]","#","'target_text","'",":","[","[","'","<","BOS",">","'",",","'target","'",",","'sequence","'",",","'","1","'",",","'","<","EOS",">","'","]","]",",","#","'target_text_ids","'",":","[","[","1",",","6",",","10",",","20",",","2","]","]",",","#","'target_length","'",":","[","5","]","#","}"]
docstr: Returns a dicitionary of default hyperparameters.
  
  .. code-block:: python
  
      {
          # (1) Hyperparams specific to text dataset
          "source_dataset": {
              "files": [],
              "compression_type": None,
              "vocab_file": "",
              "embedding_init": {},
              "delimiter": " ",
              "max_seq_length": None,
              "length_filter_mode": "truncate",
              "pad_to_max_seq_length": False,
              "bos_token": None,
              "eos_token": "<EOS>",
              "other_transformations": [],
              "variable_utterance": False,
              "utterance_delimiter": "|||",
              "max_utterance_cnt": 5,
              "data_name": "source",
          },
          "target_dataset": {
              # ...
              # Same fields are allowed as in "source_dataset" with the
              # same default values, except the
              # following new fields/values:
              "bos_token": "<BOS>"
              "vocab_share": False,
              "embedding_init_share": False,
              "processing_share": False,
              "data_name": "target"
          }
          # (2) General hyperparams
          "num_epochs": 1,
          "batch_size": 64,
          "allow_smaller_final_batch": True,
          "shuffle": True,
          "shuffle_buffer_size": None,
          "shard_and_shuffle": False,
          "num_parallel_calls": 1,
          "prefetch_buffer_size": 0,
          "max_dataset_size": -1,
          "seed": None,
          "name": "paired_text_data",
          # (3) Bucketing
          "bucket_boundaries": [],
          "bucket_batch_sizes": None,
          "bucket_length_fn": None,
      }
  
  Here:
  
  1. Hyperparameters in the :attr:`"source_dataset"` and
  attr:`"target_dataset"` fields have the same definition as those
  in :meth:`texar.tf.data.MonoTextData.default_hparams`, for source and
  target text, respectively.
  
  For the new hyperparameters in "target_dataset":
  
      "vocab_share": bool
          Whether to share the vocabulary of source.
          If `True`, the vocab file of target is ignored.
  
      "embedding_init_share": bool
          Whether to share the embedding initial value of source. If
          `True`, :attr:`"embedding_init"` of target is ignored.
  
          :attr:`"vocab_share"` must be true to share the embedding
          initial value.
  
      "processing_share": bool
          Whether to share the processing configurations of source,
          including
          "delimiter", "bos_token", "eos_token", and
          "other_transformations".
  
  2. For the **general** hyperparameters, see
  :meth:`texar.tf.data.DataBase.default_hparams` for details.
  
  3. For **bucketing** hyperparameters, see
  :meth:`texar.tf.data.MonoTextData.default_hparams` for details, except
  that the default bucket_length_fn is the maximum sequence length
  of source and target sequences.
docstr_tok: ["Returns","a","dicitionary","of","default","hyperparameters",".","..","code-block",":",":","python","{","#","(","1",")","Hyperparams","specific","to","text","dataset","``","source_dataset","''",":","{","``","files","''",":","[","]",",","``","compression_type","''",":","None",",","``","vocab_file","''",":","``","''",",","``","embedding_init","''",":","{","}",",","``","delimiter","''",":","``","``",",","``","max_seq_length","''",":","None",",","``","length_filter_mode","''",":","``","truncate","''",",","``","pad_to_max_seq_length","''",":","False",",","``","bos_token","''",":","None",",","``","eos_token","''",":","``","<","EOS",">","''",",","``","other_transformations","''",":","[","]",",","``","variable_utterance","''",":","False",",","``","utterance_delimiter","''",":","``","|||","''",",","``","max_utterance_cnt","''",":","5",",","``","data_name","''",":","``","source","''",",","}",",","``","target_dataset","''",":","{","#","...","#","Same","fields","are","allowed","as","in","``","source_dataset","''","with","the","#","same","default","values",",","except","the","#","following","new","fields\/values",":","``","bos_token","''",":","``","<","BOS",">","''","``","vocab_share","''",":","False",",","``","embedding_init_share","''",":","False",",","``","processing_share","''",":","False",",","``","data_name","''",":","``","target","''","}","#","(","2",")","General","hyperparams","``","num_epochs","''",":","1",",","``","batch_size","''",":","64",",","``","allow_smaller_final_batch","''",":","True",",","``","shuffle","''",":","True",",","``","shuffle_buffer_size","''",":","None",",","``","shard_and_shuffle","''",":","False",",","``","num_parallel_calls","''",":","1",",","``","prefetch_buffer_size","''",":","0",",","``","max_dataset_size","''",":","-1",",","``","seed","''",":","None",",","``","name","''",":","``","paired_text_data","''",",","#","(","3",")","Bucketing","``","bucket_boundaries","''",":","[","]",",","``","bucket_batch_sizes","''",":","None",",","``","bucket_length_fn","''",":","None",",","}","Here",":","1",".","Hyperparameters","in","the",":","attr",":","`","``","source_dataset","''","`","and","attr",":","`","``","target_dataset","''","`","fields","have","the","same","definition","as","those","in",":","meth",":","`","texar.tf.data.MonoTextData.default_hparams","`",",","for","source","and","target","text",",","respectively",".","For","the","new","hyperparameters","in","``","target_dataset","''",":","``","vocab_share","''",":","bool","Whether","to","share","the","vocabulary","of","source",".","If","`","True","`",",","the","vocab","file","of","target","is","ignored",".","``","embedding_init_share","''",":","bool","Whether","to","share","the","embedding","initial","value","of","source",".","If","`","True","`",",",":","attr",":","`","``","embedding_init","''","`","of","target","is","ignored",".",":","attr",":","`","``","vocab_share","''","`","must","be","true","to","share","the","embedding","initial","value",".","``","processing_share","''",":","bool","Whether","to","share","the","processing","configurations","of","source",",","including","``","delimiter","''",",","``","bos_token","''",",","``","eos_token","''",",","and","``","other_transformations","''",".","2",".","For","the","*","*","general","*","*","hyperparameters",",","see",":","meth",":","`","texar.tf.data.DataBase.default_hparams","`","for","details",".","3",".","For","*","*","bucketing","*","*","hyperparameters",",","see",":","meth",":","`","texar.tf.data.MonoTextData.default_hparams","`","for","details",",","except","that","the","default","bucket_length_fn","is","the","maximum","sequence","length","of","source","and","target","sequences","."]
code: def default_hparams():
      """Returns a dicitionary of default hyperparameters.
  
      .. code-block:: python
  
          {
              # (1) Hyperparams specific to text dataset
              "source_dataset": {
                  "files": [],
                  "compression_type": None,
                  "vocab_file": "",
                  "embedding_init": {},
                  "delimiter": " ",
                  "max_seq_length": None,
                  "length_filter_mode": "truncate",
                  "pad_to_max_seq_length": False,
                  "bos_token": None,
                  "eos_token": "<EOS>",
                  "other_transformations": [],
                  "variable_utterance": False,
                  "utterance_delimiter": "|||",
                  "max_utterance_cnt": 5,
                  "data_name": "source",
              },
              "target_dataset": {
                  # ...
                  # Same fields are allowed as in "source_dataset" with the
                  # same default values, except the
                  # following new fields/values:
                  "bos_token": "<BOS>"
                  "vocab_share": False,
                  "embedding_init_share": False,
                  "processing_share": False,
                  "data_name": "target"
              }
              # (2) General hyperparams
              "num_epochs": 1,
              "batch_size": 64,
              "allow_smaller_final_batch": True,
              "shuffle": True,
              "shuffle_buffer_size": None,
              "shard_and_shuffle": False,
              "num_parallel_calls": 1,
              "prefetch_buffer_size": 0,
              "max_dataset_size": -1,
              "seed": None,
              "name": "paired_text_data",
              # (3) Bucketing
              "bucket_boundaries": [],
              "bucket_batch_sizes": None,
              "bucket_length_fn": None,
          }
  
      Here:
  
      1. Hyperparameters in the :attr:`"source_dataset"` and
      attr:`"target_dataset"` fields have the same definition as those
      in :meth:`texar.tf.data.MonoTextData.default_hparams`, for source and
      target text, respectively.
  
      For the new hyperparameters in "target_dataset":
  
          "vocab_share": bool
              Whether to share the vocabulary of source.
              If `True`, the vocab file of target is ignored.
  
          "embedding_init_share": bool
              Whether to share the embedding initial value of source. If
              `True`, :attr:`"embedding_init"` of target is ignored.
  
              :attr:`"vocab_share"` must be true to share the embedding
              initial value.
  
          "processing_share": bool
              Whether to share the processing configurations of source,
              including
              "delimiter", "bos_token", "eos_token", and
              "other_transformations".
  
      2. For the **general** hyperparameters, see
      :meth:`texar.tf.data.DataBase.default_hparams` for details.
  
      3. For **bucketing** hyperparameters, see
      :meth:`texar.tf.data.MonoTextData.default_hparams` for details, except
      that the default bucket_length_fn is the maximum sequence length
      of source and target sequences.
  
      """
      hparams = TextDataBase.default_hparams()
      hparams["name"] = "paired_text_data"
      hparams.update(_default_paired_text_dataset_hparams())
      return hparams
  
imports: import copy
  import tensorflow
  from texar.tf.utils import utils
  from texar.tf.utils.dtypes import is_callable
  from texar.tf.data.data.mono_text_data import _default_mono_text_dataset_hparams
  from texar.tf.data.data.text_data_base import TextDataBase
  from texar.tf.data.data.mono_text_data import MonoTextData
  from texar.tf.data.data_utils import count_file_lines
  from texar.tf.data.data import dataset_utils
  from texar.tf.data.vocabulary import Vocab, SpecialTokens
  from texar.tf.data.embedding import Embedding
label_desc: Use image node_id 2 for calling the PairedTextData obj's underlying member method code with example usage: obj.default_hparams() and returns: hparams