EDIT2：
新的训练集......
输入：
[
[0.0, 0.0],
[0.0, 1.0],
[0.0, 2.0],
[0.0, 3.0],
[0.0, 4.0],
[1.0, 0.0],
[1.0, 1.0],
[1.0, 2.0],
[1.0, 3.0],
[1.0, 4.0],
[2.0, 0.0],
[2.0, 1.0],
[2.0, 2.0],
[2.0, 3.0],
[2.0, 4.0],
[3.0, 0.0],
[3.0, 1.0],
[3.0, 2.0],
[3.0, 3.0],
[3.0, 4.0],
[4.0, 0.0],
[4.0, 1.0],
[4.0, 2.0],
[4.0, 3.0],
[4.0, 4.0]
]
输出：
[
[0.0],
[0.0],
[0.0],
[0.0],
[0.0],
[0.0],
[0.0],
[0.0],
[0.0],
[0.0],
[0.0],
[0.0],
[0.0],
[0.0],
[0.0],
[0.0],
[0.0],
[0.0],
[1.0],
[1.0],
[0.0],
[0.0],
[0.0],
[1.0],
[1.0]
]
EDIT1：
我已使用我的最新代码更新了问题。我解决了一些小问题，但在网络学习之后，我仍然为所有输入组合获得相同的输出。
以下是backprop算法：Backprop algorithm
是的，这是一个功课，要在开始时明确这一点。
我应该在一个简单的神经网络上实现一个简单的反向传播算法。
我选择Python作为此任务的首选语言，我选择了这样的神经网络：
3层：1个输入，1个隐藏，1个输出层：
O O
O
O O
两个inptut神经元都有一个整数，输出神经元上有1或0。
这是我的整个实现（有点长）。 Bellow it我将选择更短的相关片段，我认为错误可能位于：
import os
import math
import Image
import random
from random import sample
#------------------------------ class definitions
class Weight:
def __init__(self, fromNeuron, toNeuron):
self.value = random.uniform(-0.5, 0.5)
self.fromNeuron = fromNeuron
self.toNeuron = toNeuron
fromNeuron.outputWeights.append(self)
toNeuron.inputWeights.append(self)
self.delta = 0.0 # delta value, this will accumulate and after each training cycle used to adjust the weight value
def calculateDelta(self, network):
self.delta += self.fromNeuron.value * self.toNeuron.error
class Neuron:
def __init__(self):
self.value = 0.0 # the output
self.idealValue = 0.0 # the ideal output
self.error = 0.0 # error between output and ideal output
self.inputWeights = []
self.outputWeights = []
def activate(self, network):
x = 0.0;
for weight in self.inputWeights:
x += weight.value * weight.fromNeuron.value
# sigmoid function
if x < -320:
self.value = 0
elif x > 320:
self.value = 1
else:
self.value = 1 / (1 + math.exp(-x))
class Layer:
def __init__(self, neurons):
self.neurons = neurons
def activate(self, network):
for neuron in self.neurons:
neuron.activate(network)
class Network:
def __init__(self, layers, learningRate):
self.layers = layers
self.learningRate = learningRate # the rate at which the network learns
self.weights = []
for hiddenNeuron in self.layers[1].neurons:
for inputNeuron in self.layers[0].neurons:
self.weights.append(Weight(inputNeuron, hiddenNeuron))
for outputNeuron in self.layers[2].neurons:
self.weights.append(Weight(hiddenNeuron, outputNeuron))
def setInputs(self, inputs):
self.layers[0].neurons[0].value = float(inputs[0])
self.layers[0].neurons[1].value = float(inputs[1])
def setExpectedOutputs(self, expectedOutputs):
self.layers[2].neurons[0].idealValue = expectedOutputs[0]
def calculateOutputs(self, expectedOutputs):
self.setExpectedOutputs(expectedOutputs)
self.layers[1].activate(self) # activation function for hidden layer
self.layers[2].activate(self) # activation function for output layer
def calculateOutputErrors(self):
for neuron in self.layers[2].neurons:
neuron.error = (neuron.idealValue - neuron.value) * neuron.value * (1 - neuron.value)
def calculateHiddenErrors(self):
for neuron in self.layers[1].neurons:
error = 0.0
for weight in neuron.outputWeights:
error += weight.toNeuron.error * weight.value
neuron.error = error * neuron.value * (1 - neuron.value)
def calculateDeltas(self):
for weight in self.weights:
weight.calculateDelta(self)
def train(self, inputs, expectedOutputs):
self.setInputs(inputs)
self.calculateOutputs(expectedOutputs)
self.calculateOutputErrors()
self.calculateHiddenErrors()
self.calculateDeltas()
def learn(self):
for weight in self.weights:
weight.value += self.learningRate * weight.delta
def calculateSingleOutput(self, inputs):
self.setInputs(inputs)
self.layers[1].activate(self)
self.layers[2].activate(self)
#return round(self.layers[2].neurons[0].value, 0)
return self.layers[2].neurons[0].value
#------------------------------ initialize objects etc
inputLayer = Layer([Neuron() for n in range(2)])
hiddenLayer = Layer([Neuron() for n in range(100)])
outputLayer = Layer([Neuron() for n in range(1)])
learningRate = 0.5
network = Network([inputLayer, hiddenLayer, outputLayer], learningRate)
# just for debugging, the real training set is much larger
trainingInputs = [
[0.0, 0.0],
[1.0, 0.0],
[2.0, 0.0],
[0.0, 1.0],
[1.0, 1.0],
[2.0, 1.0],
[0.0, 2.0],
[1.0, 2.0],
[2.0, 2.0]
]
trainingOutputs = [
[0.0],
[1.0],
[1.0],
[0.0],
[1.0],
[0.0],
[0.0],
[0.0],
[1.0]
]
#------------------------------ let's train
for i in range(500):
for j in range(len(trainingOutputs)):
network.train(trainingInputs[j], trainingOutputs[j])
network.learn()
#------------------------------ let's check
for pattern in trainingInputs:
print network.calculateSingleOutput(pattern)
现在，问题在于，在学习网络后，对于所有输入组合，似乎返回的浮点数非常接近0.0，即使是那些应该接近1.0的浮点数。
我在100个周期内训练网络，在每个周期中我都会这样做：
对于训练集中的每组输入：
然后我根据学习率和累积的增量来调整权重。
这是我对神经元的激活功能：
def activationFunction(self, network):
"""
Calculate an activation function of a neuron which is a sum of all input weights * neurons where those weights start
"""
x = 0.0;
for weight in self.inputWeights:
x += weight.value * weight.getFromNeuron(network).value
# sigmoid function
self.value = 1 / (1 + math.exp(-x))
这是我如何计算增量：
def calculateDelta(self, network):
self.delta += self.getFromNeuron(network).value * self.getToNeuron(network).error
这是我的算法的一般流程：
for i in range(numberOfIterations):
for k,expectedOutput in trainingSet.iteritems():
coordinates = k.split(",")
network.setInputs((float(coordinates[0]), float(coordinates[1])))
network.calculateOutputs([float(expectedOutput)])
network.calculateOutputErrors()
network.calculateHiddenErrors()
network.calculateDeltas()
oldWeights = network.weights
network.adjustWeights()
network.resetDeltas()
print "Iteration ", i
j = 0
for weight in network.weights:
print "Weight W", weight.i, weight.j, ": ", oldWeights[j].value, " ............ Adjusted value : ", weight.value
j += j
输出的最后两行是：
0.552785449458 # this should be close to 1
0.552785449458 # this should be close to 0
它实际上返回所有输入组合的输出数。
我错过了什么吗？
答案 0 :(得分：5)
看起来你得到的几乎是神经元的初始状态（差不多self.idealValue）。也许你不应该在提供实际数据之前初始化这个神经元？
编辑：好的，我在代码中看起来更深一些并简化了一下（将在下面发布简化版）。基本上你的代码有两个小错误（看起来像你忽略的东西），但这会导致网络无法正常工作。
显然，这两个问题中的任何一个都会导致一个不连贯的网络。
一旦纠正，它就可以了（好吧，它在我的代码的简化版本中）。
错误并不容易发现，因为初始代码太复杂了。在引入新类或新方法之前，您应该三思而后行。没有创建足够的方法或类会使代码难以阅读和维护，但创建太多可能会使其更难以阅读和维护。你必须找到合适的平衡点。我找到这种平衡的个人方法是遵循code smells并在他们引导我的地方重构技巧。有时添加方法或创建类，有时会删除它们。它当然不是完美的，但这对我有用。
在应用了一些重构之后，下面是我的代码版本。我花了大约一个小时更改你的代码，但始终保持其功能相当。我认为这是一个很好的重构练习，因为最初的代码真的很糟糕。在重构之后，它花了5分钟来发现问题。
import os
import math
"""
A simple backprop neural network. It has 3 layers:
Input layer: 2 neurons
Hidden layer: 2 neurons
Output layer: 1 neuron
"""
class Weight:
"""
Class representing a weight between two neurons
"""
def __init__(self, value, from_neuron, to_neuron):
self.value = value
self.from_neuron = from_neuron
from_neuron.outputWeights.append(self)
self.to_neuron = to_neuron
to_neuron.inputWeights.append(self)
# delta value, this will accumulate and after each training cycle
# will be used to adjust the weight value
self.delta = 0.0
class Neuron:
"""
Class representing a neuron.
"""
def __init__(self):
self.value = 0.0 # the output
self.idealValue = 0.0 # the ideal output
self.error = 0.0 # error between output and ideal output
self.inputWeights = [] # weights that end in the neuron
self.outputWeights = [] # weights that starts in the neuron
def activate(self):
"""
Calculate an activation function of a neuron which is
a sum of all input weights * neurons where those weights start
"""
x = 0.0;
for weight in self.inputWeights:
x += weight.value * weight.from_neuron.value
# sigmoid function
self.value = 1 / (1 + math.exp(-x))
class Network:
"""
Class representing a whole neural network. Contains layers.
"""
def __init__(self, layers, learningRate, weights):
self.layers = layers
self.learningRate = learningRate # the rate at which the network learns
self.weights = weights
def training(self, entries, expectedOutput):
for i in range(len(entries)):
self.layers[0][i].value = entries[i]
for i in range(len(expectedOutput)):
self.layers[2][i].idealValue = expectedOutput[i]
for layer in self.layers[1:]:
for n in layer:
n.activate()
for n in self.layers[2]:
error = (n.idealValue - n.value) * n.value * (1 - n.value)
n.error = error
for n in self.layers[1]:
error = 0.0
for w in n.outputWeights:
error += w.to_neuron.error * w.value
n.error = error
for w in self.weights:
w.delta += w.from_neuron.value * w.to_neuron.error
def updateWeights(self):
for w in self.weights:
w.value += self.learningRate * w.delta
def calculateSingleOutput(self, entries):
"""
Calculate a single output for input values.
This will be used to debug the already learned network after training.
"""
for i in range(len(entries)):
self.layers[0][i].value = entries[i]
# activation function for output layer
for layer in self.layers[1:]:
for n in layer:
n.activate()
print self.layers[2][0].value
#------------------------------ initialize objects etc
neurons = [Neuron() for n in range(5)]
w1 = Weight(-0.79, neurons[0], neurons[2])
w2 = Weight( 0.51, neurons[0], neurons[3])
w3 = Weight( 0.27, neurons[1], neurons[2])
w4 = Weight(-0.48, neurons[1], neurons[3])
w5 = Weight(-0.33, neurons[2], neurons[4])
w6 = Weight( 0.09, neurons[3], neurons[4])
weights = [w1, w2, w3, w4, w5, w6]
inputLayer = [neurons[0], neurons[1]]
hiddenLayer = [neurons[2], neurons[3]]
outputLayer = [neurons[4]]
learningRate = 0.3
network = Network([inputLayer, hiddenLayer, outputLayer], learningRate, weights)
# just for debugging, the real training set is much larger
trainingSet = [([0.0,0.0],[0.0]),
([1.0,0.0],[1.0]),
([2.0,0.0],[1.0]),
([0.0,1.0],[0.0]),
([1.0,1.0],[1.0]),
([2.0,1.0],[0.0]),
([0.0,2.0],[0.0]),
([1.0,2.0],[0.0]),
([2.0,2.0],[1.0])]
#------------------------------ let's train
for i in range(100): # training iterations
for entries, expectedOutput in trainingSet:
network.training(entries, expectedOutput)
network.updateWeights()
#network has learned, let's check
network.calculateSingleOutput((1, 0)) # this should be close to 1
network.calculateSingleOutput((0, 0)) # this should be close to 0
顺便说一下，还有第三个问题我没有纠正（但很容易纠正）。如果x太大或太小（> 320或<-320）math.exp()将引发异常。如果你申请训练迭代，比如几千，就会发生这种情况。我所看到的最简单的纠正方法是检查x的值，如果它太大或太小，将Neuron的值设置为0或1，具体取决于具体情况，即极限值。