'Constructs a model, that respects the interface required by the
trainer class.
:type cost_layer: groundhog layer
:param cost_layer: the cost (last) layer of the model
:type sample_fn: function or None
:param sample_fn: function used to sample from the model
:type valid_fn: function or None
:param valid_fn: function used to compute the validation error on a
minibatch of examples
:type noise_fn: function or None
:param noise_fn: function called to corrupt an input (that
potentially will be denoised by the model)
:type clean_before_noise_fn: bool
:param clean_before_noise_fn: If the weight noise should be removed
before calling the `noise_fn` to corrupt some input
:type clean_noise_validation: bool
:param clean_noise_validation: If the weight noise should be removed
before calling the validation function
:type weight_noise_amount: float or theano scalar
:param weight_noise_amount: weight noise scale (standard deviation
of the Gaussian from which it is sampled)
:type indx_word: string or None
:param indx_word: path to the file describing how to match indices
to words (or characters)
:type need_inputs_for_generating_noise: bool
:param need_inputs_for_generating_noise: flag saying if the shape of
the inputs affect the shape of the weight noise that is generated at
each step
:type indx_word_src: string or None
:param indx_word_src: similar to indx_word (but for the source
language
:type character_level: bool
:param character_level: flag used when sampling, saying if we are
running the model on characters or words
:type excluding_params_for_norm: None or list of theano variables
:param excluding_params_for_norm: list of parameters that should not
be included when we compute the norm of the gradient (for norm
clipping). Usually the output weights if the output layer is
large
:type rng: numpy random generator
:param rng: numpy random generator'

def __init__(self, cost_layer=None, sample_fn=None, valid_fn=None, noise_fn=None, clean_before_noise_fn=False, clean_noise_validation=True, weight_noise_amount=0, indx_word='/data/lisa/data/PennTreebankCorpus/dictionaries.npz', need_inputs_for_generating_noise=False, indx_word_src=None, character_level=False, exclude_params_for_norm=None, rng=None):

    super(LM_Model, self).__init__(output_layer=cost_layer, sample_fn=sample_fn, indx_word=indx_word, indx_word_src=indx_word_src, rng=rng)
    if (exclude_params_for_norm is None):
        self.exclude_params_for_norm = []
    else:
        self.exclude_params_for_norm = exclude_params_for_norm
    self.need_inputs_for_generating_noise = need_inputs_for_generating_noise
    self.cost_layer = cost_layer
    self.validate_step = valid_fn
    self.clean_noise_validation = clean_noise_validation
    self.noise_fn = noise_fn
    self.clean_before = clean_before_noise_fn
    self.weight_noise_amount = weight_noise_amount
    self.character_level = character_level
    self.valid_costs = ['cost', 'ppl']
    state_below = self.cost_layer.state_below
    if (hasattr(self.cost_layer, 'mask') and self.cost_layer.mask):
        div = TT.sum(self.cost_layer.mask)
    else:
        div = TT.cast(state_below.shape[0], 'float32')
    if (hasattr(self.cost_layer, 'cost_scale') and self.cost_layer.cost_scale):
        div = (div * self.cost_layer.cost_scale)
    grad_norm = TT.sqrt(sum((TT.sum((x ** 2)) for (x, p) in zip(self.param_grads, self.params) if (p not in self.exclude_params_for_norm))))
    new_train_cost = (self.train_cost / div)
    new_properties = [('grad_norm', grad_norm), ('traincost', (new_train_cost / numpy.float32(numpy.log(2)))), ('trainppl', (10 ** (new_train_cost / numpy.float32(numpy.log(10)))))]
    self.properties += new_properties
    if ((len(self.noise_params) > 0) and weight_noise_amount):
        if self.need_inputs_for_generating_noise:
            inps = self.inputs
        else:
            inps = []
        self.add_noise = theano.function(inps, [], name='add_noise', updates=[(p, self.trng.normal(shp_fn(self.inputs), avg=0, std=weight_noise_amount, dtype=p.dtype)) for (p, shp_fn) in zip(self.noise_params, self.noise_params_shape_fn)], on_unused_input='ignore')
        self.del_noise = theano.function(inps, [], name='del_noise', updates=[(p, TT.zeros(shp_fn(self.inputs), p.dtype)) for (p, shp_fn) in zip(self.noise_params, self.noise_params_shape_fn)], on_unused_input='ignore')
    else:
        self.add_noise = None
        self.del_noise = None
