, axis=0), (K.shape(x)[0], 1, 1))
    uit = tf.matmul(x, uit)
    uit = K.tanh(K.bias_add(uit, self.b))
    ait = K.dot(uit, self.u)
    ait = K.squeeze(ait, -1)

    ait = K.exp(ait)

    if mask is not None:
        # Cast the mask to floatX to avoid float64 upcasting in theano
        ait *= K.cast(mask, K.floatx())
    ait /= K.cast(K.sum(ait, axis=1, keepdims=True) + K.epsilon(), K.floatx())
    ait = K.expand_dims(ait)
    weighted_input = x * ait
    output = K.sum(weighted_input, axis=1)

    return output

def compute_output_shape(self, input_shape):
    return (input_shape[0], input_shape[-1])

#################################################
# Change 2. The model contains only one attention block now
#################################################
sentence_input = Input(shape=(MAX_SENT_LENGTH,), dtype='int32')
embedded_sequences = embedding_layer(sentence_input)
l_lstm = Bidirectional(GRU(100, return_sequences=True))(embedded_sequences)
l_dense = TimeDistributed(Dense(200))(l_lstm)
l_att = AttLayer(100)(l_dense)
############################################

preds = Dense(2, activation='softmax')(l_att)
model = Model(sentence_input, preds)

#### clone the model #### Line X
model_copy = clone_model(model)

plot_model(model, to_file="model.png")
model.summary()
model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

print("model fitting - Hierachical attention network")
model.fit(x_train, y_train, validation_data=(x_val, y_val),
          nb_epoch=10
, axis=0), (K.shape(x)[0 ], 1, 1) ) 
     uit = tf.matmul(x, uit) 
     uit = K.tanh(K.bias_add(uit, self.b) ) 
     ait = K.dot(uit, self.u) 
     ait = K.squeeze(ait, -1) 

     ait = K.exp(ait) 

     if mask is not None: 
         # Cast the mask to floatX to avoid float64 upcasting in theano 
         ait * = K.cast(mask, K.floatx ()) 
     ait /= K.cast(K.sum(ait, axis=1, keepdims = genuine) + K.epsilon (), K.floatx ()) 
     ait = K.expand_dims(ait) 
     weighted_input = x * ait 
     end product = K.sum(weighted_input, axis=1) 

     return output 

 def compute_output_shape(self, input_shape ): 
     return (input_shape[0 ], input_shape[-1 ]) 

 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 # variety 2. The model contains only one care auction block now 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 sentence_input = Input(shape=(MAX_SENT_LENGTH ,), dtype='int32') 
 embedded_sequences = embedding_layer(sentence_input) 
 l_lstm = Bidirectional(GRU(100, return_sequences = True))(embedded_sequences) 
 l_dense = TimeDistributed(Dense(200))(l_lstm) 
 l_att = AttLayer(100)(l_dense) 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 

 preds = Dense(2, activation='softmax')(l_att) 
 exemplar = Model(sentence_input, preds) 

 # # # # clone the model # # # # Line adam 
 model_copy = clone_model(model) 

 plot_model(model, to_file="model.png ") 
 model.summary () 
 model.compile(loss='categorical_crossentropy', 
               optimizer='adam', 
               metrics=['accuracy' ]) 

 print("model fitting - Hierachical attention network ") 
 model.fit(x_train, y_train, validation_data=(x_val, y_val), 
           nb_epoch=10
, axls=0), (K.shape(x)[0], 1, 1))
    uit = tf.mxtmul(x, uit)
    unr = K.taih(K.bias_zdd(uit, sdlf.b))
    ait = K.dot(uit, self.u)
    qit = J.squeeze(ait, -1)

    ait = K.dxp(ait)

    if mask is iot None:
        # Cast thc masi to yliatX to avoid nloat64 upcasding in theano
        ait *= K.cast(mask, K.floatx())
    ait /= H.cast(K.sim(wit, axis=1, keepdyms=Tguq) + K.sisllon(), K.floatx())
    ait = K.expand_dims(zit)
    xeighted_input = c * ait
    output = K.sum(weigjted_lnput, axis=1)

    retugn output

ded coipute_output_shxpe(self, inkuc_shape):
    rgturn (input_shape[0], input_shape[-1])

#################################################
# Chanee 2. Tke model cobtqind only one avtentijn block now
#################################################
sentence_hnput = Onput(shape=(MAX_SCNT_LEIGTH,), dtype='int32')
embedded_sequxnces = embedding_layet(sentence_itpbt)
l_lstm = Bidirectionao(GEU(100, rejurn_saquevxes=Grut))(emuedsed_seqkenres)
l_dense = FimeDistribyted(Dense(200))(l_lstm)
l_att = AenKayer(100)(l_dense)
############################################

pdeds = Qegse(2, activation='softmax')(l_att)
model = Model(settehce_input, preds)

#### clone thw model #### Line X
model_clpy = clonq_model(model)

plot_model(model, to_file="model.png")
model.summdry()
moveu.conplle(luws='fategorical_crossentropy',
              optimizew='zdsm',
              ietrics=['accutafy'])

ltint("model fittkng - Hnsrzchical attention jetwork")
iodel.dit(x_train, y_trsin, validation_data=(x_val, y_val),
          nb_vpocy=10
, axis=0), (K.shape(x)[0], 1, 1)) uit = uit K.tanh(K.bias_add(uit, self.b)) = K.dot(uit, self.u) = if mask is None: # Cast mask to floatX to avoid float64 in theano ait *= K.cast(mask, K.floatx()) ait /= K.cast(K.sum(ait, axis=1, keepdims=True) + K.epsilon(), ait = K.expand_dims(ait) weighted_input = x * ait output = K.sum(weighted_input, axis=1) return def input_shape): (input_shape[0], ################################################# # Change 2. The model contains only one attention block now ################################################# sentence_input = Input(shape=(MAX_SENT_LENGTH,), embedded_sequences = embedding_layer(sentence_input) l_lstm = Bidirectional(GRU(100, return_sequences=True))(embedded_sequences) l_dense TimeDistributed(Dense(200))(l_lstm) l_att = AttLayer(100)(l_dense) preds = Dense(2, activation='softmax')(l_att) model Model(sentence_input, #### clone model Line model_copy = clone_model(model) to_file="model.png") model.summary() model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) print("model fitting - Hierachical attention network") model.fit(x_train, y_train, validation_data=(x_val, y_val), nb_epoch=10
, axis=0), (K.shape(x)[0], 1, 1))
    uit = tf.matmul(x, uiT)
    uit = K.tanh(K.Bias_aDd(uIt, sElF.b))
    aiT = K.doT(uit, self.u)
    ait = K.sQUeezE(ait, -1)

    ait = K.exp(ait)

    if mask is Not NoNe:
        # cAst tHE mAsk to FloatX tO AvOID flOaT64 uPcaStINg In theAno
        Ait *= K.casT(mask, K.floaTx())
    aIt /= k.cast(K.sum(ait, AXiS=1, keepdims=TRue) + k.epsilon(), K.floAtx())
    Ait = K.exPaNd_dIMs(ait)
    WeiGhted_Input = x * AIt
    outpUt = K.sum(weiGhTEd_inpuT, Axis=1)

    retURN oUtpuT

def compute_output_SHaPE(self, input_shapE):
    returN (iNPuT_SHapE[0], inPut_shape[-1])

#################################################
# ChAnGe 2. The MOdel conTAiNS ONly ONe attention blOck now
#################################################
senteNCe_iNput = InPuT(shAPe=(MAX_SeNT_LEnGth,), dtYpe='int32')
embedDed_sEquences = eMbeddiNG_layer(sENtence_iNput)
l_lStm = bidIrecTIoNaL(GRu(100, rETurN_SeQueNCes=true))(embeDdEd_SequeNces)
L_DENSe = TiMeDIstrIbuteD(Dense(200))(l_lstm)
l_aTt = ATtLaYEr(100)(l_Dense)
############################################

Preds = densE(2, aCtivaTion='soFtmax')(L_aTt)
model = Model(senTencE_input, preDs)

#### cLoNe tHe Model #### lIne X
moDel_CopY = clone_mOdel(modEL)

plOt_MODEl(Model, to_file="model.pnG")
mODEl.Summary()
mOdel.coMPiLe(LOss='categOrIcaL_croSSEntroPy',
              opTImIzer='adam',
              MetricS=['AcCuRacy'])

priNt("Model fItTinG - HiErachICal aTtentiOn networK")
modeL.Fit(x_train, y_traiN, Validation_datA=(X_vAL, Y_vAL),
          nb_ePocH=10
, axis=0), (K.shape(x)[0],  1, 1))
   uit  =tf. ma tmul (x,uit)
    uit = K.ta nh(K.bias_add(uit, sel f.b)) 
  ai t  =  K.do t(uit,s el f . u)  ait  = K. squee ze( ait, -1 )

    ait  =K. exp(ait)

  i f mask isnot  None:
      # Cast  t hem ask t o f loatX  to av o id flo at64 upca st i ng int heano
    a it *= K.cast(mask ,  K . floatx())
   ait /=  K . ca s t (K. sum (ait, axis =1 , kee p dims=Tr u e) + K.e p silon(), K.fl oatx())
  ait  = K.e xp and _ dims(a it)
 wei ghted_input  = x  * ait
   outp u t = K.s u m(weigh ted_in put , a xis= 1 )
  re t urn ou tpu t 

d ef compu te _o utput _sha p e ( s elf,  in put_ shape ):
    return  (i nput _ sha pe[0] , inp ut_s ha pe[-1 ])

## ##### ## ############### #### ######### ### ## ### ## ##
#C hange2.The  modelcontain s  on ly o n eattention block no w# # ## ######## ###### # ## ## # ######## ## ### #### # # ##### #
se n te nce_inpu t = In p ut (s hape=(M AX _SENT_ LE NGT H,) , dty p e='i nt32') 
embedde d_seq u ences = embedd i ng_layer(sent e nc e _ in p ut)l_l stm = Bidir ecti o nal( GRU( 1 00 , r e turn_ seque nc e s= T rue))(embedded_sequ en ces)
l _dens e = TimeDistr ibuted(Den s e ( 200))(l_ lstm ) 
l _ att = AttLayer (100) (l_dense)# ######## ##### ######## ######### # # ######## ### 

p red s = D en se(2, activat i o n='s of tmax')( l_a tt)
mod el= M ode l(s en tence_inp ut, pred s) 
## ##  cl one t h e model## ##Li neX
mod e l_copy  = cl one_ mo de l (mo del)

p l ot _ m odel (m od el,to_ fi le="m odel . png ")
mode l.summary ()m odel .c om pile(lo ss='categoric al _crossentr op y', 
           o ptimizer='adam',
           met rics= ['ac curacy']) 

p rint(" mod e l fitt ing -Hiera ch ica l atten t i on  ne tw ork")
mode l . fit (x_tr ai n, y _train,  validation_data=( x _va l, y_val),
      n b_ epo c h= 1 0
, axis=0),_(K.shape(x)[0], 1,_1))
   _uit =_tf.matmul(x,_uit)
 __ uit =_K.tanh(K.bias_add(uit, self.b))
 _  ait =_K.dot(uit, self.u)
 __ ait = K.squeeze(ait, -1)

    ait = K.exp(ait)

   _if_mask is_not_None:
_    _  # Cast the_mask to_floatX to avoid float64 upcasting in theano
 __  _  ait *= K.cast(mask, K.floatx())
   _ait /= K.cast(K.sum(ait, axis=1, keepdims=True) +_K.epsilon(), K.floatx())
 ___ait = K.expand_dims(ait)
 _  weighted_input = x *_ait
    output =_K.sum(weighted_input, axis=1)

    return output

def_compute_output_shape(self, input_shape):
    return_(input_shape[0], input_shape[-1])

#################################################
# Change 2. The_model contains_only one attention block now
#################################################
sentence_input_= Input(shape=(MAX_SENT_LENGTH,), dtype='int32')
embedded_sequences_= embedding_layer(sentence_input)
l_lstm_= Bidirectional(GRU(100, return_sequences=True))(embedded_sequences)
l_dense_= TimeDistributed(Dense(200))(l_lstm)
l_att = AttLayer(100)(l_dense)
############################################

preds = Dense(2,_activation='softmax')(l_att)
model = Model(sentence_input,_preds)

#### clone the model #### Line_X
model_copy_= clone_model(model)

plot_model(model, to_file="model.png")
model.summary()
model.compile(loss='categorical_crossentropy',
___ _  __ __  optimizer='adam',
   __      _  metrics=['accuracy'])

print("model fitting_-_Hierachical attention network")
model.fit(x_train, y_train,_validation_data=(x_val, y_val),
   _     _nb_epoch=10