

こんにちは  
TransformerをPytorchを利用して実装しようとしているのですが、  
おもったような結果になりません。  
以下が、訓練のコード、モデル定義のコード、データローダと出力結果のファイルです。

訓練のコード  
**Train.py**

[code]

    from transformer import EncoderDecoder, Encoder, LayerNorm, EncoderLayer, Decoder, DecoderLayer, MultiHeadedAttention, PositionwiseFeedForward, Embeddings, PositionalEncoding, Generator
    import numpy as np
    import torch
    import torch.nn as nn
    import torch.nn.functional as F
    import math, copy, time
    from torch.autograd import Variable
    import argparse
    from loader import Seq2seqDataset
    from torchtext import data
    import sentencepiece as spm
    
    sp = spm.SentencePieceProcessor()
    sp.load("./bert-wiki-ja/wiki-ja.model")
    
    def subsequent_mask(size):
        "Mask out subsequent positions."
        attn_shape = (1, size, size)
        subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8')
        return torch.from_numpy(subsequent_mask) == 0
    
    def make_model(src_vocab, tgt_vocab, N=6, 
                   d_model=516, d_ff=2048, h=8, dropout=0.1):
        "Helper: Construct a model from hyperparameters."
        c = copy.deepcopy
        attn = MultiHeadedAttention(h, d_model)
        ff = PositionwiseFeedForward(d_model, d_ff, dropout)
        position = PositionalEncoding(d_model, dropout)
        model = EncoderDecoder(
            Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N),
            Decoder(DecoderLayer(d_model, c(attn), c(attn), 
                                 c(ff), dropout), N),
            nn.Sequential(Embeddings(d_model, src_vocab), c(position)),
            nn.Sequential(Embeddings(d_model, tgt_vocab), c(position)),
            Generator(d_model, tgt_vocab))
    
        # This was important from their code. 
        # Initialize parameters with Glorot / fan_avg.
        for p in model.parameters():
            if p.dim() > 1:
                nn.init.xavier_uniform(p)
        return model
    
    def data_load(maxlen, source_size, batch_size):
        data_set = Seq2seqDataset(maxlen=maxlen)
        data_num = len(data_set)
        train_ratio = int(data_num*0.8)
        eval_ratio = int(data_num*0.1)
        test_ratio = int(data_num*0.1)
        res = int(data_num - (train_ratio + eval_ratio + test_ratio))
        train_ratio += res
        ratio=[train_ratio, eval_ratio, test_ratio]
        train_dataset, val_dataset, test_dataset = torch.utils.data.random_split(data_set, ratio)
        dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
        val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=True)
        del(train_dataset)
        del(data_set)
        return dataloader, val_dataloader
    
    def make_std_mask(tgt, pad):
        tgt_mask = (tgt != pad).unsqueeze(-2)
        tgt_mask = tgt_mask & Variable(subsequent_mask(tgt.size(-1)).type_as(tgt_mask.data))
        return tgt_mask
    
    def run_epoch(data_iter, model, loss_compute):
        "Standard Training and Logging Function"
        start = time.time()
        total_tokens = 0
        total_loss = 0
        tokens = 0
        pad = 0
        model = model.train()
        print(len(data_iter))
        for i, data in enumerate(data_iter):
            model = model.cuda() 
            src, trg = data[0], data[1]
            src, trg = src.cuda(), trg.cuda()
            src_mask = (src != pad).unsqueeze(-2)
            trg = trg[:, :-1]
            trg_y = trg[:, 1:]
            trg_mask = make_std_mask(trg, pad)
            ntokens = (trg_y != pad).data.sum()
            out = model.forward(src, trg, 
                                src_mask, trg_mask)
            loss = loss_compute(out, trg_y, ntokens)
            total_loss += loss
            total_tokens += ntokens
            tokens += ntokens
            if i % 50 == 1:
                elapsed = time.time() - start
                print("Epoch Step: %d Loss: %f Tokens per Sec: %f" %
                        (i, loss / ntokens, tokens / elapsed))
                start = time.time()
                tokens = 0
        torch.save({
                    'model': model.state_dict()
                    }, "./model_log/transformer.pt")
        return total_loss / total_tokens
    
    global max_src_in_batch, max_tgt_in_batch
    def batch_size_fn(new, count, sofar):
        "Keep augmenting batch and calculate total number of tokens + padding."
        global max_src_in_batch, max_tgt_in_batch
        if count == 1:
            max_src_in_batch = 0
            max_tgt_in_batch = 0
        max_src_in_batch = max(max_src_in_batch,  len(new.src))
        max_tgt_in_batch = max(max_tgt_in_batch,  len(new.trg) + 2)
        src_elements = count * max_src_in_batch
        tgt_elements = count * max_tgt_in_batch
        return max(src_elements, tgt_elements)
    
    
    
    def get_std_opt(model):
        return NoamOpt(model.src_embed[0].d_model, 2, 4000,
                torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))
    
    def greedy_decode(model, data_iter, maxlen, start_symbol):
        print("------start_val-----")
        model = model.eval()
        all_y = []
        for i, data in enumerate(data_iter):
            aa = data[0]
            if i/50 == 0:
                print(str(i) + "/" + str(len(data_iter)))
            c = 0
            for src in aa:
                src = src.cuda()
                src = src.unsqueeze(0)
                src_mask = Variable(torch.ones(1,1,20))
                src_mask = src_mask.cuda()
                ys = torch.ones(1, 1).fill_(start_symbol).type_as(src.data)
                memory = model.encode(src, src_mask)
                for i in range(maxlen-1):
                    out = model.decode(memory, src_mask, 
                                    Variable(ys), 
                                    Variable(subsequent_mask(ys.size(1)).type_as(src.data)))
                    prob = model.generator(out[:, -1])
                    _, next_word = torch.max(prob, dim = 1)
                    next_word = next_word.data[0]
                    ys = torch.cat([ys, 
                                    torch.ones(1, 1).type_as(src.data).fill_(next_word)], dim=1)
                all_y.append(ys)
                if c == 1:
                    break
                c = c+1
            break
        return all_y
    
    
    class NoamOpt:
        "Optim wrapper that implements rate."
        def __init__(self, model_size, factor, warmup, optimizer):
            self.optimizer = optimizer
            self._step = 0
            self.warmup = warmup
            self.factor = factor
            self.model_size = model_size
            self._rate = 0
    
        def step(self):
            "Update parameters and rate"
            self._step += 1
            rate = self.rate()
            for p in self.optimizer.param_groups:
                p['lr'] = rate
            self._rate = rate
            self.optimizer.step()
    
        def rate(self, step = None):
            "Implement `lrate` above"
            if step is None:
                step = self._step
            return self.factor * \
                (self.model_size ** (-0.5) *
                min(step ** (-0.5), step * self.warmup ** (-1.5)))
    
    class LabelSmoothing(nn.Module):
        "Implement label smoothing."
        def __init__(self, size, padding_idx, smoothing=0.0):
            super(LabelSmoothing, self).__init__()
            self.criterion = nn.KLDivLoss(size_average=False)
            self.padding_idx = padding_idx
            self.confidence = 1.0 - smoothing
            self.smoothing = smoothing
            self.size = size
            self.true_dist = None
    
        def forward(self, x, target):
            assert x.size(1) == self.size
            true_dist = x.data.clone()
            true_dist.fill_(self.smoothing / (self.size - 2))
            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
            true_dist[:, self.padding_idx] = 0
            mask = torch.nonzero(target.data == self.padding_idx)
            if mask.dim() > 0:
                true_dist.index_fill_(0, mask.squeeze(), 0.0)
            self.true_dist = true_dist
            return self.criterion(x, Variable(true_dist, requires_grad=False))
    
    class SimpleLossCompute:
        "A simple loss compute and train function."
        def __init__(self, generator, criterion, opt=None):
            self.generator = generator
            self.criterion = criterion
            self.opt = opt
    
        def __call__(self, x, y, norm):
            x = self.generator(x)
            x = x[:,1:]
            loss = self.criterion(x.contiguous().view(-1, x.size(-1)), 
                                  y.contiguous().view(-1)) / norm
            loss.backward()
            if self.opt is not None:
                self.opt.step()
                self.opt.optimizer.zero_grad()
            return loss.data * norm
    
    if __name__ == "__main__":
        source_size = 32000
        parser = argparse.ArgumentParser(description='Parse training parameters')
        parser.add_argument('--do_train', type=str, default='False',
                            help='')
        parser.add_argument('--batch_size', type=int, default=128,
                            help='number of examples in a batch')
        parser.add_argument('--maxlen', type=int, default=20,
                            help='Sequences will be padded or truncated to this size.')    
        parser.add_argument('--epochs', type=int, default=10,
                            help='the number of epochs to train')   
        args = parser.parse_args()             
        model = make_model(source_size, source_size)
        data_iter, val_data_iter = data_load(args.maxlen, source_size, args.batch_size)
    
        model = model.train()
        model = model.cuda()
    
        V=32000
        criterion = LabelSmoothing(size=V, padding_idx=sp.PieceToId("[PAD]"), smoothing=0.0)
        model_opt = NoamOpt(model.src_embed[0].d_model, 1, 400,
            torch.optim.Adam(model.parameters(), lr=0., betas=(0.9, 0.98), eps=1e-9))
    
        if args.do_train == "True":
            for epoch in range(args.epochs):
                run_epoch(data_iter, model, SimpleLossCompute(model.generator, criterion, model_opt))
        else:
            model.load_state_dict(torch.load("./model_log/transformer.pt")["model"])
        start_symbol = sp.PieceToId("<s>")
        result = greedy_decode(model, val_data_iter, args.maxlen, start_symbol)
        print("------end_val------")
        with open("./log.txt", "w", encoding='UTF-8') as f:
            for i in  result:
                for j in i:
                    for tok in j:
                        f.write(sp.IdToPiece(int(tok)))
                    f.write("\n")
    
[/code]

モデル定義のコード  
**transformer.py**

[code]

    import numpy as np
    import torch
    import torch.nn as nn
    import torch.nn.functional as F
    import math, copy, time
    from torch.autograd import Variable
    import matplotlib.pyplot as plt
    import seaborn
    seaborn.set_context(context="talk")
    
    class EncoderDecoder(nn.Module):
        """
        A standard Encoder-Decoder architecture. Base for this and many 
        other models.
        """
        def __init__(self, encoder, decoder, src_embed, tgt_embed, generator):
            super(EncoderDecoder, self).__init__()
            self.encoder = encoder
            self.decoder = decoder
            self.src_embed = src_embed
            self.tgt_embed = tgt_embed
            self.generator = generator
    
        def forward(self, src, tgt, src_mask, tgt_mask):
            "Take in and process masked src and target sequences."
            return self.decode(self.encode(src, src_mask), src_mask,
                                tgt, tgt_mask)
    
        def encode(self, src, src_mask):
            return self.encoder(self.src_embed(src), src_mask)
    
        def decode(self, memory, src_mask, tgt, tgt_mask):
            return self.decoder(self.tgt_embed(tgt), memory, src_mask, tgt_mask)
    
    class Generator(nn.Module):
        "Define standard linear + softmax generation step."
        def __init__(self, d_model, vocab):
            super(Generator, self).__init__()
            self.proj = nn.Linear(d_model, vocab)
    
        def forward(self, x):
            return F.log_softmax(self.proj(x), dim=-1)
    
    def clones(module, N):
        "Produce N identical layers."
        return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])
    
    class Encoder(nn.Module):
        "Core encoder is a stack of N layers"
        def __init__(self, layer, N):
            super(Encoder, self).__init__()
            self.layers = clones(layer, N)
            self.norm = LayerNorm(layer.size)
    
        def forward(self, x, mask):
            "Pass the input (and mask) through each layer in turn."
            for layer in self.layers:
                x = layer(x, mask)
            return self.norm(x)
    
    class LayerNorm(nn.Module):
        "Construct a layernorm module (See citation for details)."
        def __init__(self, features, eps=1e-6):
            super(LayerNorm, self).__init__()
            self.a_2 = nn.Parameter(torch.ones(features))
            self.b_2 = nn.Parameter(torch.zeros(features))
            self.eps = eps
    
        def forward(self, x):
            mean = x.mean(-1, keepdim=True)
            std = x.std(-1, keepdim=True)
            return self.a_2 * (x - mean) / (std + self.eps) + self.b_2
    
    class SublayerConnection(nn.Module):
        """
        A residual connection followed by a layer norm.
        Note for code simplicity the norm is first as opposed to last.
        """
        def __init__(self, size, dropout):
            super(SublayerConnection, self).__init__()
            self.norm = LayerNorm(size)
            self.dropout = nn.Dropout(dropout)
    
        def forward(self, x, sublayer):
            "Apply residual connection to any sublayer with the same size."
            return x + self.dropout(sublayer(self.norm(x)))
    
    class EncoderLayer(nn.Module):
        "Encoder is made up of self-attn and feed forward (defined below)"
        def __init__(self, size, self_attn, feed_forward, dropout):
            super(EncoderLayer, self).__init__()
            self.self_attn = self_attn
            self.feed_forward = feed_forward
            self.sublayer = clones(SublayerConnection(size, dropout), 2)
            self.size = size
    
        def forward(self, x, mask):
            "Follow Figure 1 (left) for connections."
            x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, mask))
            return self.sublayer[1](x, self.feed_forward)
    
    class Decoder(nn.Module):
        "Generic N layer decoder with masking."
        def __init__(self, layer, N):
            super(Decoder, self).__init__()
            self.layers = clones(layer, N)
            self.norm = LayerNorm(layer.size)
    
        def forward(self, x, memory, src_mask, tgt_mask):
            for layer in self.layers:
                x = layer(x, memory, src_mask, tgt_mask)
            return self.norm(x)
    
    class DecoderLayer(nn.Module):
        "Decoder is made of self-attn, src-attn, and feed forward (defined below)"
        def __init__(self, size, self_attn, src_attn, feed_forward, dropout):
            super(DecoderLayer, self).__init__()
            self.size = size
            self.self_attn = self_attn
            self.src_attn = src_attn
            self.feed_forward = feed_forward
            self.sublayer = clones(SublayerConnection(size, dropout), 3)
    
        def forward(self, x, memory, src_mask, tgt_mask):
            "Follow Figure 1 (right) for connections."
            m = memory
            x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, tgt_mask))
            x = self.sublayer[1](x, lambda x: self.src_attn(x, m, m, src_mask))
            return self.sublayer[2](x, self.feed_forward)
    
    
    
    def attention(query, key, value, mask=None, dropout=None):
        "Compute 'Scaled Dot Product Attention'"
        d_k = query.size(-1)
        scores = torch.matmul(query, key.transpose(-2, -1)) \
                 / math.sqrt(d_k)
        if mask is not None:
            scores = scores.masked_fill(mask == 0, -1e9)
        p_attn = F.softmax(scores, dim = -1)
        if dropout is not None:
            p_attn = dropout(p_attn)
        return torch.matmul(p_attn, value), p_attn
    
    class MultiHeadedAttention(nn.Module):
        def __init__(self, h, d_model, dropout=0.1):
            "Take in model size and number of heads."
            super(MultiHeadedAttention, self).__init__()
            assert d_model % h == 0
            # We assume d_v always equals d_k
            self.d_k = d_model // h
            self.h = h
            self.linears = clones(nn.Linear(d_model, d_model), 4)
            self.attn = None
            self.dropout = nn.Dropout(p=dropout)
    
        def forward(self, query, key, value, mask=None):
            "Implements Figure 2"
            if mask is not None:
                # Same mask applied to all h heads.
                mask = mask.unsqueeze(1)
            nbatches = query.size(0)
    
            # 1) Do all the linear projections in batch from d_model => h x d_k 
            query, key, value = \
                [l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)
                 for l, x in zip(self.linears, (query, key, value))]
    
            # 2) Apply attention on all the projected vectors in batch. 
            x, self.attn = attention(query, key, value, mask=mask, 
                                     dropout=self.dropout)
    
            # 3) "Concat" using a view and apply a final linear. 
            x = x.transpose(1, 2).contiguous() \
                 .view(nbatches, -1, self.h * self.d_k)
            return self.linears[-1](x)
    
    class PositionwiseFeedForward(nn.Module):
        "Implements FFN equation."
        def __init__(self, d_model, d_ff, dropout=0.1):
            super(PositionwiseFeedForward, self).__init__()
            self.w_1 = nn.Linear(d_model, d_ff)
            self.w_2 = nn.Linear(d_ff, d_model)
            self.dropout = nn.Dropout(dropout)
    
        def forward(self, x):
            return self.w_2(self.dropout(F.relu(self.w_1(x))))
    
    class Embeddings(nn.Module):
        def __init__(self, d_model, vocab):
            super(Embeddings, self).__init__()
            self.lut = nn.Embedding(vocab, d_model)
            self.d_model = d_model
    
        def forward(self, x):
            return self.lut(x) * math.sqrt(self.d_model)
    
    class PositionalEncoding(nn.Module):
        "Implement the PE function."
        def __init__(self, d_model, dropout, max_len=5000):
            super(PositionalEncoding, self).__init__()
            self.dropout = nn.Dropout(p=dropout)
    
            # Compute the positional encodings once in log space.
            pe = torch.zeros(max_len, d_model)
            position = torch.arange(0.0, max_len).unsqueeze(1)
            div_term = torch.exp(torch.arange(0.0, d_model, 2) * (-(math.log(10000.0) / d_model)))
            pe[:, 0::2] = torch.sin(position * div_term)
            pe[:, 1::2] = torch.cos(position * div_term)
            pe = pe.unsqueeze(0)
            self.register_buffer('pe', pe)
    
        def forward(self, x):
            x = x + Variable(self.pe[:, :x.size(1)], 
                             requires_grad=False)
            return self.dropout(x)
    
    def make_model(src_vocab, tgt_vocab, N=6, 
                   d_model=512, d_ff=2048, h=8, dropout=0.1):
        "Helper: Construct a model from hyperparameters."
        c = copy.deepcopy
        attn = MultiHeadedAttention(h, d_model)
        ff = PositionwiseFeedForward(d_model, d_ff, dropout)
        position = PositionalEncoding(d_model, dropout)
        model = EncoderDecoder(
            Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N),
            Decoder(DecoderLayer(d_model, c(attn), c(attn), 
                                 c(ff), dropout), N),
            nn.Sequential(Embeddings(d_model, src_vocab), c(position)),
            nn.Sequential(Embeddings(d_model, tgt_vocab), c(position)),
            Generator(d_model, tgt_vocab))
    
        # This was important from their code. 
        # Initialize parameters with Glorot / fan_avg.
        for p in model.parameters():
            if p.dim() > 1:
                nn.init.xavier_uniform(p)
        return model
    
    
[/code]

データローダ  
**loader.py**

[code]

    import torch
    import numpy as np
    import csv
    import sentencepiece as spm
    
    class Seq2seqDataset(torch.utils.data.Dataset):
        def __init__(self, maxlen):
            self.sp = spm.SentencePieceProcessor()
            self.sp.load("./bert-wiki-ja/wiki-ja.model")
            self.maxlen = maxlen
    
    
            with open('./data/parallel_data.csv', mode='r', newline='', encoding='utf-8') as f:
                csv_file = csv.reader(f)
                read_data = [row for row in csv_file]
            self.data_num = len(read_data) - 1
            jp_data = []
            es_data = []
            for i in range(1, self.data_num):    
                jp_data.append(read_data[i][1:2])
                es_data.append(read_data[i][2:3])
    
    
            self.en_data_idx = np.zeros((len(jp_data), maxlen))
            self.de_data_idx = np.zeros((len(es_data), maxlen+1))
    
            for i,sentence in enumerate(jp_data):
                for j,idx in enumerate(self.sp.EncodeAsIds(sentence[0])[1:]):
                    self.en_data_idx[i][j] = idx
                    if j == maxlen-1:
                        break
                if j < maxlen-1:
                    self.en_data_idx[i][j:] = self.sp.PieceToId("[PAD]")
            for i,sentence in enumerate(es_data):
                self.de_data_idx[i][0] = self.sp.PieceToId("<s>")
                for j,idx in enumerate(self.sp.EncodeAsIds(sentence[0])[1:]):
                    self.de_data_idx[i][j+1] = idx
                    if j+1 == maxlen:
                        break
                if j+1 < maxlen:
                    self.de_data_idx[i][j+1:] = self.sp.PieceToId("[PAD]")
    
        def __len__(self):
            return self.data_num
    
        def __getitem__(self, idx):
            en_data = torch.tensor(self.en_data_idx[idx-1][:], dtype=torch.long)
            de_data = torch.tensor(self.de_data_idx[idx-1][:], dtype=torch.long)
            target = torch.zeros((self.maxlen+1), dtype=torch.long)
            for i, data in enumerate(de_data[:]):
                target[i] = data
            target[i] = self.sp.PieceToId("</s>")
    
            return en_data, target
    
[/code]

出力結果  
**log.txt**

[code]

    <s>ををををををををををををををををををを
    <s>ををををををををををををををををををを
    
[/code]

入力データにはhttp://www.jnlp.org/SNOW/T15のxlsxファイルをcsvに変換して使用しました。  
難しい日本語をエンコーダへの入力に、優しい日本語をデコーダへの入力にしています。  
形態素解析に用いたsentencepieceモデルはbertの事前学習に利用されたhttps://yoheikikuta.github.io/bert-
japanese/を利用しています。  
transformerのモデルはhttp://nlp.seas.harvard.edu/2018/04/03/attention.htmlを参考にほぼ丸写しで実装しています。  
ハイパーパラメータはTrain.pyを引数なし（ただし--do_train="True"）で実行した場合で訓練しました。

また過学習を疑い、件数が20倍のデータ（公開できません）で学習したり、モデルの層を減らしたりもしてみたのですがほとんど同じ結果でした。

原因はどこにあるのでしょうか？


CC BY-SA 4.0
2020-05-05T07:29:06.103
66315
2020-05-05T07:29:06.103
36311
question
python 深層学習 pytorch
PytorchでのTransformerの実装に関して