nprnn.py
def forward_backward(inputs, targets, initial_states):
	'''
	Computes forward and backward pass through the recurrent net, for SEQ_SIZE time steps
	-inputs is an array of shape [BATCH_SIZE, SEQ_SIZE, VOCABULARY_SIZE] and holds one hot encoded inputs to the model
	-targets has a shape [BATCH_SIZE, SEQ_SIZE], holds just the indices of the target chars
	-initial_states contains state of all the recurrent hidden units, shape [DEPTH, BATCH_SIZE, HIDDEN_LAYER_SIZE]

	Returns loss, gradients and the last state of the hidden units
	'''
	loss = 0

	dropout = [{} for i in xrange(DEPTH)]
	x,h,z = [{} for i in xrange(DEPTH + 1)], [{} for i in xrange(DEPTH)], {}
	
	# Initialize states
	h = [{-1: initial_states[d]} for d in xrange(DEPTH)]

	# Forward pass
	for t in xrange(SEQ_SIZE):
		x[0][t] = np.reshape(inputs[:,t,:], (BATCH_SIZE, VOCABULARY_SIZE))
			
		for d in xrange(DEPTH):
			dropout[d][t] = np.random.binomial(1, 1 - DROPOUT_RATE, (1, HIDDEN_LAYER_SIZE)) * 1.0 / (1 - DROPOUT_RATE)
			h[d][t] = relu(np.dot(x[d][t], Wxh[d]) + np.dot(h[d][t - 1], Whh[d]) + bh[d])
			x[d + 1][t] = np.copy(h[d][t]) * dropout[d][t]
		y = np.dot(x[DEPTH][t], Why) + by
		y = np.clip(y, -100,100) # clipping to prevent state explosions at the beggining of training
		z[t] = np.array([softmax(y[b,:]) for b in xrange(BATCH_SIZE)])

	# Backward pass
	dWhy = np.zeros((HIDDEN_LAYER_SIZE, VOCABULARY_SIZE))
	dby = np.zeros((1, VOCABULARY_SIZE))
	dWxh = [np.zeros((VOCABULARY_SIZE if d == 0 else HIDDEN_LAYER_SIZE, HIDDEN_LAYER_SIZE)) for d in xrange(DEPTH)]
	dWhh = [np.zeros((HIDDEN_LAYER_SIZE, HIDDEN_LAYER_SIZE)) for d in xrange(DEPTH)]
	dbh = [np.zeros((1, HIDDEN_LAYER_SIZE)) for d in xrange(DEPTH)]
	dhh = [np.zeros((BATCH_SIZE, HIDDEN_LAYER_SIZE)) for i in xrange(DEPTH)]
	
	for t in reversed(xrange(SEQ_SIZE)):
		gt = targets[:, t]

		loss += np.array([cross_entropy(z[t][b, :], gt[b]) for b in xrange(BATCH_SIZE)]).sum() / (SEQ_SIZE * BATCH_SIZE)
		dy = np.array([cross_entropy_softmax_d(z[t][b, :], gt[b]) for b in xrange(BATCH_SIZE)]) / (SEQ_SIZE * BATCH_SIZE)
		
		dWhy += np.dot(x[DEPTH][t].T, dy)
		dby += dy.sum(0)
		dh = np.dot(dy, Why.T) 
		for d in reversed(xrange(DEPTH)):
			dhinput = relu_d(np.dot(x[d][t], Wxh[d]) + np.dot(h[d][t-1], Whh[d]) + bh[d]) * (dh * dropout[d][t] + dhh[d])
			dWxh[d] += np.dot(x[d][t].T, dhinput)
			dWhh[d] += np.dot(h[d][t-1].T, dhinput)
			dbh[d] += dhinput.sum(0)
			dhh[d] = np.dot(dhinput, Whh[d].T)
			dh = np.dot(dhinput, Wxh[d].T)
	
	h_prev = np.array([h[d][SEQ_SIZE - 1] for d in xrange(DEPTH)]) # get last states for the next train step
	return loss, dWxh, dWhh, dbh, dWhy, dby, h_prev

	Computes forward and backward pass through the recurrent net, for SEQ_SIZE time steps
	-inputs is an array of shape [BATCH_SIZE, SEQ_SIZE, VOCABULARY_SIZE] and holds one hot encoded inputs to the model
	-targets has a shape [BATCH_SIZE, SEQ_SIZE], holds just the indices of the target chars
	-initial_states contains state of all the recurrent hidden units, shape [DEPTH, BATCH_SIZE, HIDDEN_LAYER_SIZE]

	Returns loss, gradients and the last state of the hidden units
	
Computes forward and backward pass through the recurrent net, for SEQ_SIZE time steps
inputs is an array of shape [BATCH_SIZE, SEQ_SIZE, VOCABULARY_SIZE] and holds one hot encoded inputs to the model
targets has a shape [BATCH_SIZE, SEQ_SIZE], holds just the indices of the target chars
initial_states contains state of all the recurrent hidden units, shape [DEPTH, BATCH_SIZE, HIDDEN_LAYER_SIZE]

Returns loss, gradients and the last state of the hidden units
Computes forward and backward pass through the recurrent net for SEQ_SIZE time steps inputs is an array of shape [ BATCH_SIZE SEQ_SIZE VOCABULARY_SIZE ] and holds one hot encoded inputs to the model targets has a shape [ BATCH_SIZE SEQ_SIZE ] holds just the indices of the target chars initial_states contains state of all the recurrent hidden units shape [ DEPTH BATCH_SIZE HIDDEN_LAYER_SIZE ] Returns loss gradients and the last state of the hidden units
def forward_backward(inputs, targets, initial_states):
	loss = 0
	dropout = [{} for i in xrange(DEPTH)]
	x,h,z = [{} for i in xrange(DEPTH + 1)], [{} for i in xrange(DEPTH)], {}
	h = [{-1: initial_states[d]} for d in xrange(DEPTH)]
	for t in xrange(SEQ_SIZE):
		x[0][t] = np.reshape(inputs[:,t,:], (BATCH_SIZE, VOCABULARY_SIZE))
		for d in xrange(DEPTH):
			dropout[d][t] = np.random.binomial(1, 1 - DROPOUT_RATE, (1, HIDDEN_LAYER_SIZE)) * 1.0 / (1 - DROPOUT_RATE)
			h[d][t] = relu(np.dot(x[d][t], Wxh[d]) + np.dot(h[d][t - 1], Whh[d]) + bh[d])
			x[d + 1][t] = np.copy(h[d][t]) * dropout[d][t]
		y = np.dot(x[DEPTH][t], Why) + by
		y = np.clip(y, -100,100) 
		z[t] = np.array([softmax(y[b,:]) for b in xrange(BATCH_SIZE)])
	dWhy = np.zeros((HIDDEN_LAYER_SIZE, VOCABULARY_SIZE))
	dby = np.zeros((1, VOCABULARY_SIZE))
	dWxh = [np.zeros((VOCABULARY_SIZE if d == 0 else HIDDEN_LAYER_SIZE, HIDDEN_LAYER_SIZE)) for d in xrange(DEPTH)]
	dWhh = [np.zeros((HIDDEN_LAYER_SIZE, HIDDEN_LAYER_SIZE)) for d in xrange(DEPTH)]
	dbh = [np.zeros((1, HIDDEN_LAYER_SIZE)) for d in xrange(DEPTH)]
	dhh = [np.zeros((BATCH_SIZE, HIDDEN_LAYER_SIZE)) for i in xrange(DEPTH)]
	for t in reversed(xrange(SEQ_SIZE)):
		gt = targets[:, t]
		loss += np.array([cross_entropy(z[t][b, :], gt[b]) for b in xrange(BATCH_SIZE)]).sum() / (SEQ_SIZE * BATCH_SIZE)
		dy = np.array([cross_entropy_softmax_d(z[t][b, :], gt[b]) for b in xrange(BATCH_SIZE)]) / (SEQ_SIZE * BATCH_SIZE)
		dWhy += np.dot(x[DEPTH][t].T, dy)
		dby += dy.sum(0)
		dh = np.dot(dy, Why.T) 
		for d in reversed(xrange(DEPTH)):
			dhinput = relu_d(np.dot(x[d][t], Wxh[d]) + np.dot(h[d][t-1], Whh[d]) + bh[d]) * (dh * dropout[d][t] + dhh[d])
			dWxh[d] += np.dot(x[d][t].T, dhinput)
			dWhh[d] += np.dot(h[d][t-1].T, dhinput)
			dbh[d] += dhinput.sum(0)
			dhh[d] = np.dot(dhinput, Whh[d].T)
			dh = np.dot(dhinput, Wxh[d].T)
	h_prev = np.array([h[d][SEQ_SIZE - 1] for d in xrange(DEPTH)]) 
	return loss, dWxh, dWhh, dbh, dWhy, dby, h_prev
def forward_backward ( inputs , targets , initial_states ) : loss = 0 dropout = [ { } for i in xrange ( DEPTH ) ] x , h , z = [ { } for i in xrange ( DEPTH + 1 ) ] , [ { } for i in xrange ( DEPTH ) ] , { } h = [ { - 1 : initial_states [ d ] } for d in xrange ( DEPTH ) ] for t in xrange ( SEQ_SIZE ) : x [ 0 ] [ t ] = np . reshape ( inputs [ : , t , : ] , ( BATCH_SIZE , VOCABULARY_SIZE ) ) for d in xrange ( DEPTH ) : dropout [ d ] [ t ] = np . random . binomial ( 1 , 1 - DROPOUT_RATE , ( 1 , HIDDEN_LAYER_SIZE ) ) * 1.0 / ( 1 - DROPOUT_RATE ) h [ d ] [ t ] = relu ( np . dot ( x [ d ] [ t ] , Wxh [ d ] ) + np . dot ( h [ d ] [ t - 1 ] , Whh [ d ] ) + bh [ d ] ) x [ d + 1 ] [ t ] = np . copy ( h [ d ] [ t ] ) * dropout [ d ] [ t ] y = np . dot ( x [ DEPTH ] [ t ] , Why ) + by y = np . clip ( y , - 100 , 100 ) z [ t ] = np . array ( [ softmax ( y [ b , : ] ) for b in xrange ( BATCH_SIZE ) ] ) dWhy = np . zeros ( ( HIDDEN_LAYER_SIZE , VOCABULARY_SIZE ) ) dby = np . zeros ( ( 1 , VOCABULARY_SIZE ) ) dWxh = [ np . zeros ( ( VOCABULARY_SIZE if d == 0 else HIDDEN_LAYER_SIZE , HIDDEN_LAYER_SIZE ) ) for d in xrange ( DEPTH ) ] dWhh = [ np . zeros ( ( HIDDEN_LAYER_SIZE , HIDDEN_LAYER_SIZE ) ) for d in xrange ( DEPTH ) ] dbh = [ np . zeros ( ( 1 , HIDDEN_LAYER_SIZE ) ) for d in xrange ( DEPTH ) ] dhh = [ np . zeros ( ( BATCH_SIZE , HIDDEN_LAYER_SIZE ) ) for i in xrange ( DEPTH ) ] for t in reversed ( xrange ( SEQ_SIZE ) ) : gt = targets [ : , t ] loss += np . array ( [ cross_entropy ( z [ t ] [ b , : ] , gt [ b ] ) for b in xrange ( BATCH_SIZE ) ] ) . sum ( ) / ( SEQ_SIZE * BATCH_SIZE ) dy = np . array ( [ cross_entropy_softmax_d ( z [ t ] [ b , : ] , gt [ b ] ) for b in xrange ( BATCH_SIZE ) ] ) / ( SEQ_SIZE * BATCH_SIZE ) dWhy += np . dot ( x [ DEPTH ] [ t ] . T , dy ) dby += dy . sum ( 0 ) dh = np . dot ( dy , Why . T ) for d in reversed ( xrange ( DEPTH ) ) : dhinput = relu_d ( np . dot ( x [ d ] [ t ] , Wxh [ d ] ) + np . dot ( h [ d ] [ t - 1 ] , Whh [ d ] ) + bh [ d ] ) * ( dh * dropout [ d ] [ t ] + dhh [ d ] ) dWxh [ d ] += np . dot ( x [ d ] [ t ] . T , dhinput ) dWhh [ d ] += np . dot ( h [ d ] [ t - 1 ] . T , dhinput ) dbh [ d ] += dhinput . sum ( 0 ) dhh [ d ] = np . dot ( dhinput , Whh [ d ] . T ) dh = np . dot ( dhinput , Wxh [ d ] . T ) h_prev = np . array ( [ h [ d ] [ SEQ_SIZE - 1 ] for d in xrange ( DEPTH ) ] ) return loss , dWxh , dWhh , dbh , dWhy , dby , h_prev
Computes forward and backward pass through the recurrent net, for SEQ_SIZE time steps
inputs is an array of shape [BATCH_SIZE, SEQ_SIZE, VOCABULARY_SIZE] and holds one hot encoded inputs to the model
targets has a shape [BATCH_SIZE, SEQ_SIZE], holds just the indices of the target chars
initial_states contains state of all the recurrent hidden units, shape [DEPTH, BATCH_SIZE, HIDDEN_LAYER_SIZE]
Computes forward and backward pass through the recurrent net for SEQ_SIZE time steps inputs is an array of shape [ BATCH_SIZE SEQ_SIZE VOCABULARY_SIZE ] and holds one hot encoded inputs to the model targets has a shape [ BATCH_SIZE SEQ_SIZE ] holds just the indices of the target chars initial_states contains state of all the recurrent hidden units shape [ DEPTH BATCH_SIZE HIDDEN_LAYER_SIZE ]
'''
	Computes forward and backward pass through the recurrent net, for SEQ_SIZE time steps
	-inputs is an array of shape [BATCH_SIZE, SEQ_SIZE, VOCABULARY_SIZE] and holds one hot encoded inputs to the model
	-targets has a shape [BATCH_SIZE, SEQ_SIZE], holds just the indices of the target chars
	-initial_states contains state of all the recurrent hidden units, shape [DEPTH, BATCH_SIZE, HIDDEN_LAYER_SIZE]

	Returns loss, gradients and the last state of the hidden units
	''' # Initialize states # Forward pass # clipping to prevent state explosions at the beggining of training # Backward pass # get last states for the next train step
inputs
targets
initial_states