code: """Data generator generates batches of inputs and outputs/labels for training.
  
  The inputs are each made from two parts. The first maxlend words are the original description, followed by `eos` followed by the headline which we want to predict, except for the last word in the headline which is always `eos` and then `empty` padding until `maxlen` words.
  
  For each, input, the output is the headline words (without the start `eos` but with the ending `eos`) padded with `empty` words up to `maxlenh` words. The output is also expanded to be y-hot encoding of each word.
  
  To be more realistic, the second part of the input should be the result of generation and not the original headline.
  Instead we will flip just `nflips` words to be from the generator, but even this is too hard and instead
  implement flipping in a naive way (which consumes less time.) Using the full input (description + eos + headline) generate predictions for outputs. For nflips random words from the output, replace the original word with the word with highest probability from the prediction.
  """
  import random
  
  import numpy as np
  from keras.preprocessing import sequence
  from keras.utils import np_utils
  
  from constants import empty, eos, maxlend, maxlenh, maxlen, seed
  from sample_gen import vocab_fold, lpadd
  
  
  def flip_headline(x, nflips, model, debug, oov0, idx2word):
      """Flip some of the words in the second half (headline) with words predicted by the model."""
      if nflips is None or model is None or nflips <= 0:
          return x
  
      batch_size = len(x)
      assert np.all(x[:, maxlend] == eos)
      probs = model.predict(x, verbose=0, batch_size=batch_size)
      x_out = x.copy()
      for b in range(batch_size):
          # pick locations we want to flip
          # 0...maxlend-1 are descriptions and should be fixed
          # maxlend is eos and should be fixed
          flips = sorted(random.sample(range(maxlend + 1, maxlen), nflips))
          if debug and b < debug:
              print(b)
          for input_idx in flips:
              if x[b, input_idx] == empty or x[b, input_idx] == eos:
                  continue
              # convert from input location to label location
              # the output at maxlend (when input is eos) is feed as input at maxlend+1
              label_idx = input_idx - (maxlend + 1)
              prob = probs[b, label_idx]
              w = prob.argmax()
              if w == empty:  # replace accidental empty with oov
                  w = oov0
              if debug and b < debug:
                  print('{} => {}'.format(idx2word[x_out[b, input_idx]], idx2word[w]),)
              x_out[b, input_idx] = w
          if debug and b < debug:
              print()
      return x_out
  
  
  def conv_seq_labels(xds, xhs, nflips, model, debug, oov0, glove_idx2idx, vocab_size, nb_unknown_words, idx2word):
      """Convert description and hedlines to padded input vectors; headlines are one-hot to label."""
      batch_size = len(xhs)
      assert len(xds) == batch_size
      x = [
          vocab_fold(lpadd(xd) + xh, oov0, glove_idx2idx, vocab_size, nb_unknown_words)
          for xd, xh in zip(xds, xhs)]  # the input does not have 2nd eos
      x = sequence.pad_sequences(x, maxlen=maxlen, value=empty, padding='post', truncating='post')
      x = flip_headline(x, nflips=nflips, model=model, debug=debug, oov0=oov0, idx2word=idx2word)
  
      y = np.zeros((batch_size, maxlenh, vocab_size))
      for i, xh in enumerate(xhs):
          xh = vocab_fold(xh, oov0, glove_idx2idx, vocab_size, nb_unknown_words) + [eos] + [empty] * maxlenh  # output does have a eos at end
          xh = xh[:maxlenh]
          y[i, :, :] = np_utils.to_categorical(xh, vocab_size)
  
      return x, y
  
  
  def gen(Xd, Xh, batch_size, nb_batches, nflips, model, debug, oov0, glove_idx2idx, vocab_size, nb_unknown_words, idx2word):
      """Yield batches.
  
      for training use nb_batches=None
      for validation generate deterministic results repeating every nb_batches
      """
      # while training it is good idea to flip once in a while the values of the headlines from the
      # value taken from Xh to value generated by the model.
      c = nb_batches if nb_batches else 0
      while True:
          xds = []
          xhs = []
          if nb_batches and c >= nb_batches:
              c = 0
          new_seed = random.randint(0, 2e10)
          random.seed(c + 123456789 + seed)
          for b in range(batch_size):
              t = random.randint(0, len(Xd) - 1)
  
              xd = Xd[t]
              s = random.randint(min(maxlend, len(xd)), max(maxlend, len(xd)))
              xds.append(xd[:s])
  
              xh = Xh[t]
              s = random.randint(min(maxlenh, len(xh)), max(maxlenh, len(xh)))
              xhs.append(xh[:s])
  
          # undo the seeding before we yield inorder not to affect the caller
          c += 1
          random.seed(new_seed)
  
          yield conv_seq_labels(
              xds,
              xhs,
              nflips=nflips,
              model=model,
              debug=debug,
              oov0=oov0,
              glove_idx2idx=glove_idx2idx,
              vocab_size=vocab_size,
              nb_unknown_words=nb_unknown_words,
              idx2word=idx2word,
          )
  
apis: numpy.all numpy.zeros