How to create RNN layer on top of BERT multilingual in pytorch
I am working on a classification problem. I want to pass the BERT embedding to RNN layer and then FCN layer at the end for classification. But I am facing some issues, is there anyone who have worked on the same problem.
I created this class as below
class BERTClass(torch.nn.Module):
    def __init__(self):
        super(BERTClass, self).__init__()
        self.l1 = BertModel.from_pretrained('bert-base-multilingual-cased', return_dict=False)
        # for param in self.l1.parameters():
        #   param.requires_grad = False
        self.l2 = torch.nn.Dropout(0.4)
        self.l3 = torch.nn.RNN(768, 1028)
        self.activation = torch.nn.ReLU()
        self.l4 = torch.nn.Dropout(0.2)
        self.l5 = torch.nn.Linear(1028, 128)
        self.activation2 = torch.nn.ReLU()
        self.l6 = torch.nn.Linear(128, 10)
        
    
    def forward(self, ids, mask, token_type_ids):
        _, output_1= self.l1(ids, attention_mask = mask, token_type_ids = token_type_ids)
        output_2 = self.l2(output_1)
        output3 = self.l3(output_2)
        act = self.activation(output3)
        output4 = self.l4(act)
        output5 = self.l5(output4)
        act2 = self.activation2(output5)
        output6 = self.l6(act2)
        return output6

model = BERTClass()

but I am getting an error
&lt;ipython-input-23-bbe09bd88901&gt; in forward(self, ids, mask, token_type_ids)
     22         output_2 = self.l2(output_1)
     23         output3 = self.l3(output_2)
---&gt; 24         act = self.activation(output3)
     25         output4 = self.l4(act)
     26         output5 = self.l5(output4)

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1108         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109                 or _global_forward_hooks or _global_forward_pre_hooks):
-&gt; 1110             return forward_call(*input, **kwargs)
   1111         # Do not call functions when jit is used
   1112         full_backward_hooks, non_full_backward_hooks = [], []

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/activation.py in forward(self, input)
     96 
     97     def forward(self, input: Tensor) -&gt; Tensor:
---&gt; 98         return F.relu(input, inplace=self.inplace)
     99 
    100     def extra_repr(self) -&gt; str:

/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py in relu(input, inplace)
   1440         result = torch.relu_(input)
   1441     else:
-&gt; 1442         result = torch.relu(input)
   1443     return result
   1444 

TypeError: relu(): argument 'input' (position 1) must be Tensor, not tuple


The output of torch.nn.RNN is a tuple of shape (output, h_n) (for more info about this layer, visit this link)
So the input of activation layer in your code, should be just first element of RNN output (output3[0]).
Final code would be:
def forward(self, ids, mask, token_type_ids):
    _, output_1= self.l1(ids, attention_mask = mask, token_type_ids = token_type_ids)
    output_2 = self.l2(output_1)
    output3 = self.l3(output_2)
    act = self.activation(output3[0])
    output4 = self.l4(act)
    output5 = self.l5(output4)
    act2 = self.activation2(output5)
    output6 = self.l6(act2)
    return output6

