计算时出现梯度误差 - pytorch

Question

我正在学习使用pytorch（0.4.0）来自动化渐变计算，但是我不太明白如何使用backward（）和grad，因为我正在做练习我需要计算df / dw使用pytorch和  分析得到衍生物，分别返回auto_grad，user_grad，但我不太了解自动微分的使用，在代码中我做了f.backward（）并做了w.grad找到df / dw，另外两个计算是不对应，如果我甚至误导了衍生品，它遵循我正在使用的图表和我想要做的代码：

import numpy as np
import torch
import torch.nn.functional as F

def graph2(W_np, x_np, b_np):
    W = torch.Tensor(W_np)
    W.requires_grad = True
    x = torch.Tensor(x_np)
    b = torch.Tensor(b_np)
    u = torch.matmul(W, x) + b
    g = F.sigmoid(u)
    f = torch.sum(g)
    user_grad = (sigmoid(W_np*x_np + b_np)*(1 - sigmoid(W_np*x_np + b_np))).T*x_np
    f.backward(retain_graph=True)
    auto_grad = W.grad

    print(auto_grad) 
    print(user_grad)



 #   raise NotImplementedError("falta completar a função graph2")
    # END YOUR CODE
    return f, auto_grad, user_grad

试验：

iterations = 1000
sizes = np.random.randint(2,10, size=(iterations))
for i in range(iterations):
    size = sizes[i]
    W_np = np.random.rand(size, size)
    x_np = np.random.rand(size, 1)
    b_np = np.random.rand(size, 1)
    f, auto_grad, user_grad = graph2(W_np, x_np, b_np)
    manual_f = np.sum(sigmoid(np.matmul(W_np, x_np) + b_np))
    assert np.isclose(f.data.numpy(), manual_f, atol=1e-4), "f not correct"
    assert np.allclose(auto_grad.numpy(), user_grad), "Gradient not correct"

Answer 1

我认为你以错误的方式计算了渐变。试试这个。

import numpy as np
import torch
from torch.autograd import Variable
import torch.nn.functional as F

def sigmoid(x):
    return 1.0 / (1.0 + np.exp(-x))

def graph2(W_np, x_np, b_np):
    W = Variable(torch.Tensor(W_np), requires_grad=True)
    x = torch.tensor(x_np, requires_grad=True).type(torch.FloatTensor)
    b = torch.tensor(b_np, requires_grad=True).type(torch.FloatTensor)
    u = torch.matmul(W, x) + b
    g = F.sigmoid(u)
    f = torch.sum(g)
    user_grad = (sigmoid(np.matmul(W_np, x_np) + b_np)*(1 - sigmoid(np.matmul(W_np, x_np) + b_np)))*x_np.T
    f.backward(retain_graph=True)
    auto_grad = W.grad
    print("auto_grad", auto_grad) 
    print("user_grad", user_grad)
    # END YOUR CODE
    return f, auto_grad, user_grad




iterations = 1000
sizes = np.random.randint(2,10, size=(iterations))
for i in range(iterations):
    size = sizes[i]
    print("i, size", i, size)
    W_np = np.random.rand(size, size)
    x_np = np.random.rand(size, 1)
    b_np = np.random.rand(size, 1)
    f, auto_grad, user_grad = graph2(W_np, x_np, b_np)
    manual_f = np.sum(sigmoid(np.matmul(W_np, x_np) + b_np))
    assert np.isclose(f.data.numpy(), manual_f, atol=1e-4), "f not correct"
    assert np.allclose(auto_grad.numpy(), user_grad), "Gradient not correct"