Question

我正在尝试实现值迭代算法。我有一个网格

grid = [[0, 0, 0, +1],
    [0, "W", 0, -1],
    [0, 0, 0, 0]]

动作列表

actlist = {UP:1, DOWN:2, LEFT:3, RIGHT:4}

还有奖励功能

reward = [[0, 0, 0, 0],
      [0, 0, 0, 0],
      [0, 0, 0,  0]]

我编写了一个函数T，该函数返回3个元组的元组。

def T(i,j,actions):
if(i == 0 and j == 0):
    if(actions == UP):
        return (i,i,0.8),(i,i,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i+1,j,0.8),(i,j,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i,j,0.1),(i+1,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i,i,0.1),(i+1,j,0.1)
elif (i == 0 and j == 1):
    if(actions == UP):
        return (i,i,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i,j,0.1),(i,j,0.1)
elif(i == 0 and j == 2):
    if(actions == UP):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return(i+1,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i,j,0.1),(i+1,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i,j,0.1),(i+1,j,0.1)
elif(i == 0 and j == 3):
    if(actions == UP):
        return (-1,-1,0.8),(-1,-1,0.1),(-1,-1,0.1)
    elif(actions == DOWN):
        return (-1,-1,0.8),(-1,-1,0.1),(-1,-1,0.1)
    elif(actions == LEFT):
        return (-1,-1,0.8),(-1,-1,0.1),(-1,-1,0.1)
    elif(actions == RIGHT):
        return (-1,-1,0.8),(-1,-1,0.1),(-1,-1,0.1)
# 2nd row
elif (i == 1 and j == 0):
    if(actions == UP):
        return (i-1,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == DOWN):
        return (i+1,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i-1,j,0.1),(i+1,j,0.1)
    elif(actions == RIGHT):
        return (i,j,0.8),(i-1,j,0.1),(i+1,j,0.1)
elif(i == 1 and j ==1):
    if(actions == UP):
        return (i,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j,0.8),(i,j,0.1),(i,j,0.1)
elif (i == 1 and j == 2):
    if(actions == UP):
        return (i-1,j,0.8),(i,j,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i+1,j,0.8),(i,j,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i-1,j,0.1),(i+1,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i-1,j,0.1),(i+1,j,0.1)
elif(i == 1 and j == 3):
    if(actions == UP):
        return (-2,-2,0.8),(-2,-2,0.1),(-2,-2,0.1)
    elif(actions == DOWN):
        return (-2,-2,0.8),(-2,-2,0.1),(-2,-2,0.1)
    elif(actions == LEFT):
        return (-2,-2,0.8),(-2,-2,0.1),(-2,-2,0.1)
    elif(actions == RIGHT):
        return (-2,-2,0.8),(-2,-2,0.1),(-2,-2,0.1)      
# 3rd row
elif(i == 2 and j == 0):
    if(actions == UP):
        return (i-1,j,0.8),(i,j,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j,0.1),(i,j+1,1,0.1)
    elif(actions == LEFT):
        return (i,j,0.8),(i-1,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i-1,j,0.1),(i,j,0.1)
elif (i == 2 and j == 1):
    if(actions == UP):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i,j,0.1),(i,j,0.1)
elif(i == 2 and j == 2):
    if(actions == UP):
        return (i-1,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j-1,0.1),(i,j+1,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i-1,j,0.1),(i,j,1)
    elif(actions == RIGHT):
        return (i,j+1,0.8),(i-1,j,0.1),(i,j,0.1)
elif(i == 2 and j == 3):
    if(actions == UP):
        return (i-1,j,0.8),(i,j-1,0.1),(i,j,0.1)
    elif(actions == DOWN):
        return (i,j,0.8),(i,j-1,0.1),(i,j,0.1)
    elif(actions == LEFT):
        return (i,j-1,0.8),(i-1,j,0.1),(i,j,0.1)
    elif(actions == RIGHT):
        return (i,j,0.8),(i-1,j,0.1),(i,j,0.1)

在值迭代函数中调用此函数：

def value_iteration():
U1 = [[0, 0, 0, 0],
      [0, 0, 0, 0],
      [0, 0, 0, 0]] 
while True:
    U=U1.copy()
    delta = 0
    for i in range(len(grid)):
        for j in range(len(grid[i])):
            U1[i][j] = max(sum(p*(R(k,l)+gamma*U[k][l]) for (k,l,p) in T(i,j,a)) for a in actlist)
            print(i,j,U1[i][j],U[i][j])
            delta = max(delta, abs(U1[i][j] - U[i][j]))
    if delta <= epsilon*(1 - gamma)/gamma:
        return U

我正在更新

U=U1.copy()

在while循环中。

问题是，输出看起来像这样：

0 0 0.0 0.0
0 1 0.0 0.0
0 2 0.0 0.0
0 3 1.0 1.0
1 0 0.0 0.0
1 2 0.0 0.0
1 3 -1.0 -1.0
2 0 0.0 0.0
2 1 0.0 0.0
2 2 0.7000000000000001 0.7000000000000001
2 3 0.9630000000000001 0.9630000000000001

但是我没有在for循环中更新U。 U应该保持不变（这意味着全为零），而U1仅应该改变。为什么在for循环中U会自动设置为U1的值？

Answer 1

U1（和U）是列表列表，实际上是列表引用列表。

您正在（浅）复制外部列表，但是副本的内容仍然引用相同的内部列表。

尝试：

import copy
U = copy.deepcopy(U1)

，看看会发生什么。 deepcopy将正确地递归复制列表。

Answer 2

nimish的答案可能是最Python的，但是如果您需要创建列表的副本，也可以将其解压缩到新数组中：

U = U1[:]

哪个将创建一个新对象，不应引用旧对象

mylist = [[1,1,1],[2,2,2],[3,3,3]]
otherlist= mylist[:]
otherlist[0] = [5,5,5]
mylist
# [[1, 1, 1], [2, 2, 2], [3, 3, 3]]

值在for循环内自动更新

2 个答案: