docstr: Tests to ensure the sequences are normalized by the correct values. The end token is
  included in the length. The start token is not.
docstr_tok: ["Tests","to","ensure","the","sequences","are","normalized","by","the","correct","values",".","The","end","token","is","included","in","the","length",".","The","start","token","is","not","."]
code: def test_length_normalized_sequence_log_prob_scorer(self):
      """
      Tests to ensure the sequences are normalized by the correct values. The end token is
      included in the length. The start token is not.
      """
      self.beam_search.final_sequence_scorer = (
          LengthNormalizedSequenceLogProbabilityScorer()
      )
      expected_log_probs = np.log(np.array([0.4, 0.3, 0.2]))
      length_normalization = np.array([5, 4, 3])
      expected_scores = expected_log_probs / length_normalization
      self._check_results(expected_log_probs=expected_scores)
  
      # Introduce a length penalty
      length_penalty = 2.0
      self.beam_search.final_sequence_scorer = (
          LengthNormalizedSequenceLogProbabilityScorer(
              length_penalty=length_penalty
          )
      )
      expected_log_probs = np.log(np.array([0.4, 0.3, 0.2]))
      length_normalization = np.array(
          [
              5**length_penalty,
              4**length_penalty,
              3**length_penalty,
          ]
      )
      expected_scores = expected_log_probs / length_normalization
      self._check_results(expected_log_probs=expected_scores)
  
      # Pick a length penalty so extreme that the order of the sequences is reversed
      length_penalty = -2.0
      self.beam_search.final_sequence_scorer = (
          LengthNormalizedSequenceLogProbabilityScorer(
              length_penalty=length_penalty
          )
      )
      expected_top_k = np.array(
          [[3, 4, 5, 5, 5], [2, 3, 4, 5, 5], [1, 2, 3, 4, 5]]
      )
      expected_log_probs = np.log(np.array([0.2, 0.3, 0.4]))
      length_normalization = np.array(
          [
              3**length_penalty,
              4**length_penalty,
              5**length_penalty,
          ]
      )
      expected_scores = expected_log_probs / length_normalization
      self._check_results(
          expected_top_k=expected_top_k,
          expected_log_probs=expected_scores,
      )
  
      # Here, we set the max_steps = 4. This prevents the first sequence from finishing,
      # so its length does not include the end token, whereas the other sequences do.
      length_penalty = 2.0
      self.beam_search.max_steps = 4
      self.beam_search.final_sequence_scorer = (
          LengthNormalizedSequenceLogProbabilityScorer(
              length_penalty=length_penalty
          )
      )
      expected_top_k = np.array(
          [[1, 2, 3, 4], [2, 3, 4, 5], [3, 4, 5, 5]]
      )
      expected_log_probs = np.log(np.array([0.4, 0.3, 0.2]))
      length_normalization = np.array(
          [
              4**length_penalty,
              4**length_penalty,
              3**length_penalty,
          ]
      )
      expected_scores = expected_log_probs / length_normalization
      self._check_results(
          expected_top_k=expected_top_k,
          expected_log_probs=expected_scores,
      )
  
imports: from typing import Dict, Tuple, Union
  import numpy
  import pytest
  import torch
  from allennlp.common.checks import ConfigurationError
  from allennlp.common.testing import AllenNlpTestCase
  from allennlp.nn.beam_search import MultinomialSampler, BeamSearch, TopKSampler, TopPSampler, GumbelSampler, LengthNormalizedSequenceLogProbabilityScorer, RepeatedNGramBlockingConstraint, StepFunctionTypeWithTimestep, StepFunctionTypeNoTimestep
  from allennlp.common.params import Params
  from allennlp.nn.util import min_value_of_dtype
label_desc: Use image node_id 28 for calling the BeamSearchTest obj's underlying member method code with example usage: obj.test_length_normalized_sequence_log_prob_scorer() without return types