Question

我在pacman机器人上编写了一些RL行为，并且在我的一个函数 arg_allmax 或 chooseAction

中将我的一个列表搞砸了。

以下是我班级的代码：

package rl;

import java.util.ArrayList;
import java.util.Hashtable;

public class Qlearn {
    private double epsilon = 0.1; // Epsilon parameter for the Epsilon Greedy Strategy 
    private double alpha = 0.2; // Alpha parameter: used to influence o the refresh of Q
    private double gamma = 0.9; // used to notice or not the feedback of the next action ; if =0 -> no feed back

private int actions[];
private Hashtable< Tuple<Integer,Integer>, Double> q; // Q(s,a) : hashTable : <state,action> -> value of q


public Qlearn(int[] actions) {
    this.actions = actions;
    q = new Hashtable< Tuple<Integer,Integer>, Double>();
}

public Qlearn(int[] actions, double epsilon, double alpha, double gamma) {
    this.actions = actions;
    this.epsilon = epsilon;
    this.alpha = alpha;
    this.gamma = gamma;
    q = new Hashtable< Tuple<Integer,Integer>, Double>();
}

public Double getQ(int id_state, int id_action) {
    // get the value of Q for the state of id_state and the action id_action ( return 0 if the value is not in the hashtable ) 
    Tuple<Integer,Integer> t = new Tuple<Integer,Integer> (id_state, id_action); // we creatte a new integer object Tubple with the value of id_state and id_action 
    Double v = q.get(t);
    if(v != null) return v;
    else return 0.0;
}

// get the argmax of a list
public int argmax(double[] list) {
    int arg=-1;
    double max= 0;
    for ( int i = 0; i<list.length; i++){
        if ( list[i]>max ){
            max = list[i];
            arg = i;
        }
    }
    return arg;
}

// get all the argmax if the argmax has several iterations
public ArrayList<Integer> arg_allmax(double[] list) {
    ArrayList<Integer> args = new ArrayList<Integer>();
    int a = argmax(list);
    for ( int i = 0; i< list.length; i++){
        if (list[i] == list[a]){
            args.add(i);
        }
    }
    return args;
}

// get the max of the list
public double max(double[] list) {
    double max_ = -1e20;
    int a = argmax(list);
    max_ = list[a];
    return max_;
}


/*
 * Fonction that updates the hashtable
 *      for the action  id_action and the state  id_state
 *      if Q(s,a) had an old value, we allocate it the new value+ alpha(value - old_value)
 *      if Q(s,a) had not an old value : we allocate reward
 */
public void learnQ(int id_state, int id_action, double reward, double value) {
    Tuple<Integer,Integer> t = new Tuple<Integer,Integer>(id_state,id_action);
    Double oldv = q.get(t);

    if(oldv == null) {

        q.put(t, reward);
    } else {

        q.put(t, oldv+alpha*(value-oldv));
    }
}

/*
 * Here is the Epsilon Greedy strategy
 *      with proba epsilon :we choose a random action
 *      avec proba 1-eps : we choose the most favorable action in fonction of  Q(s,a)
 */
public int chooseAction(int id_state) {
    int action = -1;
    if(Math.random() < epsilon) {

        int i = (int)(Math.random()*actions.length);
        action = actions[i];

    } else { 
        double[] tab = new double[actions.length];
        ArrayList<Integer> argmaxarray = new ArrayList<Integer>();
        for ( int i=0; i>actions.length; i++){
            tab[i]=actions[i];
        }
        argmaxarray=arg_allmax(tab);
        int i=(int)(Math.random()*argmaxarray.size());
        action=argmaxarray.get(i);

    }

    return action;
}


/*
 * Learning after the occurence of a move
 *      1) get the most profitable potential action from  Q(s',a)
 *      2) call learnQ
 */
public void learn(int id_state1, int id_action1, double reward, int id_state2) {
    int futureAction = 0;
    futureAction = chooseAction(id_state2);
    double maxqnew = 0; // REMPLIR  
    maxqnew = getQ(futureAction, id_state2);


    learnQ(id_state1, id_action1, reward, reward + gamma*maxqnew);

}

// Affiche Q(s,a)
private void printQvalue(int id_state) {
    for(int action : actions) {
        Tuple<Integer,Integer> t = new Tuple<Integer,Integer>(id_state,action);
        Double v = q.get(t);
        System.out.print(v+" ");
    }
    System.out.println();
}

以下是eclipse告诉我的事情：

Exception in thread "AWT-EventQueue-0" java.lang.ArrayIndexOutOfBoundsException: -1
    at rl.Qlearn.arg_allmax(Qlearn.java:54)
    at rl.Qlearn.chooseAction(Qlearn.java:108)
    at rl.Qlearn.learn(Qlearn.java:138)

我认为它来自于使用all_argmax函数的chooseAction方法的else，但我找不到确切的错误！

以下是两个涉及的方法（因此它对你来说更具可读性）：

all_argmax：

public ArrayList<Integer> arg_allmax(double[] list) {
    ArrayList<Integer> args = new ArrayList<Integer>();
    int a = argmax(list);
    for ( int i = 0; i< list.length; i++){
        if (list[i] == list[a]){
            args.add(i);
        }
    }
    return args;
}

chooseAction：

public int chooseAction(int id_state) {
    int action = -1;
    if(Math.random() < epsilon) {

        int i = (int)(Math.random()*actions.length);
        action = actions[i];

    } else { 
        double[] tab = new double[actions.length];
        ArrayList<Integer> argmaxarray = new ArrayList<Integer>();
        for ( int i=0; i>actions.length; i++){
            tab[i]=actions[i];
        }
        argmaxarray=arg_allmax(tab);
        int i=(int)(Math.random()*argmaxarray.size());
        action=argmaxarray.get(i);

    }

    return action;
}

Answer 1

由于您的IndexOutOfBoundsException方法而导致argmax([])发生，原因是因为数组为空，或者因为列表中的所有双打都是负数。

在这两种情况下，int arg = -1变量永远不会设置为-1以外的其他值，由于-1不是有效的数组位置，因此在任何情况下都明显超出范围。

最好的做法是在将数组传递给argmax之前检查数组是否为空，或者在执行任何操作之前检查返回值是否有效（不是-1）。并将double max = 0更改为double max = Double.NEGATIVE_INFINITY

ArrayIndexOutOfBoundsException异常：-1

1 个答案: