Question

我试图编写自己的mnist示例，该示例可以使用一台机器的所有两个gpu。

这是一个简单的多层感知器。

这是我的代码。你可以直接运行它。

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)

import tensorflow as tf

learning_rate = 0.001
training_steps = 100000
batch_size = 100
display_step = 100

n_hidden_1 = 256
n_hidden_2 = 256
n_input = 784
n_classes = 10

def _variable_on_cpu(name, shape, initializer):
    with tf.device('/cpu:0'):
        dtype = tf.float32
        var = tf.get_variable(name, shape, initializer=initializer, dtype=dtype)
    return var

def build_model():

    def multilayer_perceptron(x, weights, biases):
        layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
        layer_1 = tf.nn.relu(layer_1)

        layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
        layer_2 = tf.nn.relu(layer_2)

        out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
        return out_layer

    with tf.variable_scope('aaa'):
        weights = {
        'h1': _variable_on_cpu('h1',[n_input, n_hidden_1],tf.constant_initializer(0.0)),
        'h2': _variable_on_cpu('h2',[n_hidden_1, n_hidden_2],tf.constant_initializer(0.0)),
        'out': _variable_on_cpu('out_w',[n_hidden_2, n_classes],tf.constant_initializer(0.0))
          }
        biases = {
        'b1': _variable_on_cpu('b1',[n_hidden_1],tf.constant_initializer(0.0)),
        'b2': _variable_on_cpu('b2',[n_hidden_2],tf.constant_initializer(0.0)),
        'out': _variable_on_cpu('out_b',[n_classes],tf.constant_initializer(0.0))
          }

        pred = multilayer_perceptron(x, weights, biases)

        cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
    return cost


def average_gradients(tower_grads):
  average_grads = []
  for grad_and_vars in zip(*tower_grads):
    grads = []
    for g,_ in grad_and_vars:
      expanded_g = tf.expand_dims(g, 0)
      grads.append(expanded_g)
    grad = tf.concat(axis=0, values=grads)
    grad = tf.reduce_mean(grad, 0)
    v = grad_and_vars[0][1]
    grad_and_var = (grad, v)
    average_grads.append(grad_and_var)
  return average_grads


with tf.Graph().as_default(), tf.device('/cpu:0'):
    x = tf.placeholder("float", [None, n_input])
    y = tf.placeholder("float", [None, n_classes])
    tower_grads = []
    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
    with tf.variable_scope(tf.get_variable_scope()):
      for i in xrange(2):
        with tf.device('/gpu:%d' % i):
                cost = build_model()
                tf.get_variable_scope().reuse_variables()
                grads = optimizer.compute_gradients(cost)
                tower_grads.append(grads)

    grads = average_gradients(tower_grads)
    apply_gradient_op = optimizer.apply_gradients(grads)
    train_op = apply_gradient_op

    init = tf.global_variables_initializer()
    sess = tf.Session()
    sess.run(init)

    for step in range(training_steps):
            image_batch, label_batch = mnist.train.next_batch(batch_size)
            _, cost_print = sess.run([train_op, cost],
                                     {x:image_batch,
                                      y:label_batch})

            if step % display_step == 0:
                print("step=%04d" % (step+1)+  " cost=" + str(cost_print))
    print("Optimization Finished!")

    sess.close()

打印信息如下：

step=0001 cost=2.30258
step=0101 cost=2.30246
step=0201 cost=2.30128
step=0301 cost=2.30376
step=0401 cost=2.29817
step=0501 cost=2.2992
step=0601 cost=2.3104
step=0701 cost=2.29995
step=0801 cost=2.29802
step=0901 cost=2.30524
step=1001 cost=2.29673
step=1101 cost=2.30016
step=1201 cost=2.31057
step=1301 cost=2.29815
step=1401 cost=2.29669
step=1501 cost=2.30345
step=1601 cost=2.29811
step=1701 cost=2.30867
step=1801 cost=2.30757
step=1901 cost=2.29716
step=2001 cost=2.30394

损失不会减少。我不知道如何解决它。

顺便说一句，GPU-Util大约是26％和26％。如何增加GPU-Util？

Answer 1

问题是，

我应该tf.constant_initializer(0.1)使用weights代替tf.constant_initializer(0)

tensorflow multi-gpu mnist例子，损失不减少

1 个答案: