Question

我正在做一个在PyTorch中标记学习模型的序列。我有两个句子，我正在分类它们是否有必要（SNLI数据集）。我将两个50个单词的句子连接在一起（有时是填充的）到一个长度为100的向量中。然后我将迷你语发送到单词嵌入中 - ＆gt; LSTM - ＆gt;线性层。我正在做交叉熵损失，但我需要[mini_batch，C]的向量进入CrossEntropyLoss函数。相反，我的矢量中仍有100个单词为[mini_batch，100，C]

这是我的模特：

class myLSTM(nn.Module):
    def __init__(self, h_size=128, v_size=10, embed_d=300, mlp_d=256):
        super(myLSTM, self).__init__()
        self.embedding = nn.Embedding(v_size, embed_d)
        self.lstm = nn.LSTM(embed_d, h_size, num_layers=1, bidirectional=True, batch_first=True)
        self.mlp = nn.Linear(mlp_d, 1024)

        # Set static embedding vectors
        self.embedding.weight.requires_grad = False

        #self.sm = nn.CrossEntropyLoss()

    def display(self):
        for param in self.parameters():
            print(param.data.size())

    def filter_params(self):
        # Might not be compatible with python 3
        #self.parameters = filter(lambda p: p.requires_grad, self.parameters())
        pass

    def init_hidden(self):
        # Need to init hidden weights in LSTM
        pass

    def forward(self, sentence):
        print(sentence.size())
        embeds = self.embedding(sentence)
        print(embeds.size())
        out, _ = self.lstm(embeds)
        print(out.size())
        out = self.mlp(out)
        return out

我的训练序列与输出：

batch_size = 3
SGD_optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=0.01, weight_decay=1e-4)
ADM_optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.01)

criterion = nn.CrossEntropyLoss()
num_epochs = 50

from torch.autograd import Variable
from torch import optim

for epoch in range(num_epochs):
    print("Epoch {0}/{1}: {2}%".format(epoch, num_epochs, float(epoch)/num_epochs))
    for start, end in tqdm(batch_index_gen(batch_size, len(n_data))):

        # Convert minibatch to numpy
        s1, s2, y = convert_to_numpy(n_data[start:end])

        # Convert numpy to Tensor
        res = np.concatenate((s1,s2), axis=1) # Attach two sentences into 1 input vector
        input_tensor = torch.from_numpy(res).type(torch.LongTensor)
        target_tensor = torch.from_numpy(y).type(torch.FloatTensor)
        data, target = Variable(input_tensor), Variable(target_tensor)

        # Zero gradients
        SGD_optimizer.zero_grad()

        # Forward Pass
        output = model.forward(data) 
        print("Output size: ")
        print(output.size())
        print("Target size: ")
        print(target.size())
        # Calculate loss with respect to training labels
        loss = criterion(output, target)

        # Backprogogate and update optimizer
        loss.backward()
        SGD_optimizer.step()
        #ADAM_optimizer.step()

输出：

Epoch 0/50: 0.0%
torch.Size([3, 100])
torch.Size([3, 100, 300])
torch.Size([3, 100, 256])
Output size: 
torch.Size([3, 100, 1024])
Target size: 
torch.Size([3])

错误：

ValueError: Expected 2 or 4 dimensions (got 3)

已编辑------------------------------------------- ------------------------

我现在接受了模特训练，但准确度不高。我的LSTM输出是否存在连接问题，然后压缩到较小的张量以通过我的线性层？

新模式：

class myLSTM(nn.Module):
    def __init__(self, h_size=128, v_size=10, embed_d=300, mlp_d=256, num_classes=3, lstm_layers=1):
        super(myLSTM, self).__init__()
        self.num_layers = lstm_layers
        self.hidden_size = h_size
        self.embedding = nn.Embedding(v_size, embed_d)
        self.lstm = nn.LSTM(embed_d, h_size, num_layers=lstm_layers, bidirectional=True, batch_first=True)
        self.mlp = nn.Linear(2 * h_size * 2, num_classes)

        # Set static embedding vectors
        self.embedding.weight.requires_grad = False

    def forward(self, s1, s2):
        # Set initial states
        #h0 = Variable(torch.zeros(self.num_layers*2, s1.size(0), self.hidden_size)).cuda() # 2 for bidirection 
        #c0 = Variable(torch.zeros(self.num_layers*2, s1.size(0), self.hidden_size)).cuda()

        batch_size = s1.size()[0]
        embeds_1 = self.embedding(s1)
        embeds_2 = self.embedding(s2)
        _, (h_1_last, _) = self.lstm(embeds_1)#, (h0, c0)) #note the change here. Last hidden state is taken
        _, (h_2_last, _) = self.lstm(embeds_2)#, (h0, c0))
        concat = torch.cat( (h_1_last, h_2_last), dim=2) #double check the dimension
        concat = concat.view(batch_size, -1)
        scores = self.mlp(concat)
        return scores

新培训

batch_size = 64
SGD_optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=0.001, weight_decay=1e-4)

criterion = nn.CrossEntropyLoss()
num_epochs = 10
model.train()

if cuda:
    model = model.cuda()
    criterion = criterion.cuda()

from torch.autograd import Variable
from torch import optim

epoch_losses = []

for epoch in range(num_epochs):
    print("Epoch {0}/{1}: {2}%".format(epoch, num_epochs, 100*float(epoch)/num_epochs))

    # Batch loss aggregator
    losses = []

    for start, end in tqdm(batch_index_gen(batch_size, len(n_data))):
        # Convert minibatch to numpy
        s1, s2, y = convert_to_numpy(n_data[start:end])

        # Convert numpy to Tensor
        s1_tensor = torch.from_numpy(s1).type(torch.LongTensor)
        s2_tensor = torch.from_numpy(s2).type(torch.LongTensor)
        target_tensor = torch.from_numpy(y).type(torch.LongTensor)

        s1 = Variable(s1_tensor)
        s2 = Variable(s2_tensor)
        target = Variable(target_tensor)

        if cuda:
            s1 = s1.cuda()
            s2 = s2.cuda()
            target = target.cuda()

        # Zero gradients
        SGD_optimizer.zero_grad()

        # Forward Pass
        output = model.forward(s1,s2) 

        # Calculate loss with respect to training labels
        loss = criterion(output, target)
        losses.append(loss.data[0])

        # Backprogogate and update optimizer
        loss.backward()
        SGD_optimizer.step()

    # concat losses to epoch losses
    epoch_losses += losses

打印张量大小的训练：

Epoch 0/10: 0.0%
Batch size: 64
Sentences
torch.Size([64, 50])
torch.Size([64, 50])
torch.Size([64, 50, 300])
torch.Size([64, 50, 300])
Hidden states
torch.Size([2, 64, 128])
torch.Size([2, 64, 128])
Concatenated hidden states
torch.Size([2, 64, 256])
Reshaped tensors for linear layer
torch.Size([64, 512])
Linear propogation
torch.Size([64, 3])

评价

def eval_model(model, mode='dev'):
    file_name = 'snli_1.0/snli_1.0_dev.jsonl' if mode == 'dev' else 'snli_1.0/snli_1.0_test.jsonl'

    dev_data, _ = obtain_data(file_name)
    dev_n_data = vocab.process_data(dev_data)

    print("Length of data: {}".format(len(dev_n_data)))

    eval_batch_size = 1024
    model.eval()

    total = len(dev_n_data)
    hit = 0
    correct = 0

    # Batch dev eval
    for start, end in batch_index_gen(eval_batch_size, len(dev_n_data)):

        s1, s2, y = convert_to_numpy(dev_n_data[start:end])

        s1_tensor = torch.from_numpy(s1).type(torch.LongTensor)
        s2_tensor = torch.from_numpy(s2).type(torch.LongTensor)
        target_tensor = torch.from_numpy(y).type(torch.LongTensor)

        s1 = Variable(s1_tensor, volatile=True)
        s2 = Variable(s2_tensor, volatile=True)
        target = Variable(target_tensor, volatile=True)

        if cuda:
            s1 = s1.cuda()
            s2 = s2.cuda()
            target = target.cuda()

        output = model.forward(s1,s2)
        loss = criterion(output, target)

        #print("output size: {}".format(output.size()))
        #print("target size: {}".format(target.size()))
        pred = output.data.max(1)[1] # get the index of the max log-probability
        #print(pred[:5])
        #print(output[:])
        correct += pred.eq(target.data).cpu().sum()

    return correct / float(total)

eval_model(model)

Answer 1

我认为你试图解决蕴涵问题的方式存在问题。

也许你可以这样做：

设计模块以接受两个句子作为输入
将这两个嵌入嵌入
使用LSTM模块对它们进行编码。
现在你有两个句子的两个固定长度矢量表示。最简单的事情就是将它们连接起来在一起。
在顶部添加衬垫层以评估每个蕴涵等级的分数（我猜3）
应用softmax以获得适当的概率分布

所以你的模型看起来像这样（仔细检查尺寸）：

class myLSTM(nn.Module):
    def __init__(self, h_size=128, v_size=10, embed_d=300, num_classes = 3):
        super(myLSTM, self).__init__()
        self.embedding = nn.Embedding(v_size, embed_d)
        self.lstm = nn.LSTM(embed_d, h_size, num_layers=1, bidirectional=True, batch_first=True)
        self.mlp = nn.Linear(2*h_size*2, num_classes) #<- change here

    def forward(self, sentence1, sentence2):
        embeds_1 = self.embedding(sentence1)
        embeds_2 = self.embedding(sentence2)
        _, (h_1_last, _) = self.lstm(embeds_1) #note the change here. Last hidden state is taken
        _, (h_2_last, _) = self.lstm(embeds_2)
        concat = torch.concat([h_1_last, h_2_last], dim=1) #double check the dimension
        scores = self.mlp(concat)
        probas = F.softmax(scores) #from torch.functional ...

然后你可以玩添加更多的隐藏层，或者思考如何以更智能的方式（注意力等）完成两个句子的组合。仔细检查CrossEntropyLoss接受的输入和目标并进行调整（是非标准化的课程分数还是概率分布）。检查http://pytorch.org/docs/master/nn.html#lstm是否有LSTM模块文档，以阐明LSTM返回的内容（您是否需要每个单词的隐藏状态或最后一个单词后的表示）。

PyTorch中的损失维度问题（序列到标签学习）

1 个答案: