Question

我用一个单层写了一个简单的MLP。当使用sigmoid激活学习XOR函数时，损失一直在减少。但是，如果我将XOR数据的标签从[0,1]更改为[-1,1]并使用tanh激活，则损失不会减少。它应该仍然可以通过简单地更改标签和使用tanh，不应该吗？如果是这样，我的实施在哪里出现了问题？

import sys

import numpy as np


class DataSet:

    def __init__(self, data):
        self.size = len(data)
        self.inputs, self.labels = [], []
        for i in range(len(data)):
            self.inputs.append(data[i][0])
            self.labels.append(data[i][1])
        try:
            self.numInputs = len(self.inputs[0])
        except TypeError:
            self.numInputs = 1
        try:
            self.numOutputs = len(self.labels[0])
        except TypeError:
            self.numOutputs = 1


class MLP:

    def __init__(self, numInputs, numHidden, numOutputs, activationFunction):
        # MLP architecture sizes
        self.numInputs = numInputs
        self.numHidden = numHidden
        self.numOutputs = numOutputs
        self.activationFunction = activationFunction.lower()

        # MLP weights
        self.IH_weights = np.random.rand(numInputs, numHidden)      # Input -> Hidden
        self.HO_weights = np.random.rand(numHidden, numOutputs)     # Hidden -> Output

        # Gradients corresponding to weight matrices computed during backprop
        self.IH_gradients = np.zeros_like(self.IH_weights)
        self.HO_gradients = np.zeros_like(self.HO_weights)

        # Input, hidden and output neuron values
        self.I = np.zeros(numInputs)    # Inputs
        self.L = np.zeros(numOutputs)   # Labels
        self.H = np.zeros(numHidden)    # Hidden
        self.O = np.zeros(numOutputs)   # Output

    # ADD BIAS FOR RELU ########################################################
    # ADD SOFTMAX ##############################################################
    def activation(self, x, derivative=False):
        if self.activationFunction == 'sigmoid':
            if derivative:
                return x * (1 - x)
            return 1 / (1 + np.exp(-x))

        if self.activationFunction == 'tanh':
            if derivative:
                return 1. - np.tanh(x) ** 2
            return np.tanh(x)

        # if self.activationFunction == 'softmax':
        #     if derivative:
        #         return ########
        #     return np.exp(x) / np.sum(np.exp(x))

        if self.activationFunction == 'relu':
            if derivative:
                return (x > 0).astype(float)
            return np.maximum(0, x)

        print("ERROR: Activation function not found.")
        sys.exit()

    def forward(self, inputs):
        # Ensure that inputs is a list
        try:
            len(inputs)
        except TypeError:
            inputs = [inputs]

        self.I = np.array(inputs).reshape(1, self.numInputs)
        self.H = self.I.dot(self.IH_weights)
        self.H = self.activation(self.H)
        self.O = self.H.dot(self.HO_weights)
        self.O = self.activation(self.O)
        return self.O, self.L

    def backwards(self, labels):
        # Ensure that labels is a list
        try:
            len(labels)
        except TypeError:
            labels = [labels]

        self.L = np.array(labels)
        self.O_error = self.L - self.O
        self.O_delta = self.O_error * self.activation(self.O, derivative=True)

        self.H_error = self.O_delta.dot(self.HO_weights.T)
        self.H_delta = self.H_error * self.activation(self.H, derivative=True)

        self.IH_gradients += self.I.T.dot(self.H_delta)
        self.HO_gradients += self.H.T.dot(self.O_delta)

        return np.sum(self.O_error ** 2)

    def updateWeights(self, learningRate):
        self.IH_weights += self.IH_gradients
        self.HO_weights += self.HO_gradients
        self.IH_gradients = np.zeros_like(self.IH_weights)
        self.HO_gradients = np.zeros_like(self.HO_weights)

# data = DataSet([
#     [[0, 0], 0],
#     [[0, 1], 1],
#     [[1, 0], 1],
#     [[1, 1], 0]
# ])
#
# mlp = MLP(data.numInputs, 20, data.numOutputs, 'sigmoid')

data = DataSet([
    [[0, 0], -1],
    [[0, 1], 1],
    [[1, 0], 1],
    [[1, 1], -1]
])

mlp = MLP(data.numInputs, 20, data.numOutputs, 'tanh')

numEpochs = 10000
learningRate = 0.01

losses = []
for epoch in range(numEpochs):
    errors = []
    correct = []
    for i in range(data.size):
        mlp.forward(data.inputs[i])
        errors.append(mlp.backwards(data.labels[i]))
    mlp.updateWeights(learningRate)
    epochLoss = np.mean(errors)
    losses.append(epochLoss)
    if epoch % 100 == 0 or epoch == numEpochs - 1:
        print("EPOCH:", epoch)
        print("LOSS: ", epochLoss, "\n")

Answer 1

您的网络未能预测的样本是（[0,0]， - 1），这是缺乏偏见的结果。对于I = [0,0]，我们有H =点（Wh，I）= [0,0，... 0]。由于tanh（0）= 0，网络无法在任何层学习任何零到非零的变换。增加偏见将解决问题;通过在填充了任何常量的数据集中添加第3列，可以实现hacky解决方案，以模拟偏差的影响。

多层感知器在S形激活下工作正常，但不能用双曲正切

1 个答案: