Question

我试图重现DNN，其中使用了称为BlockRelu的块激活功能。定义为

我试图根据一些有关自定义激活函数的示例代码来编写此函数，但是这些函数都是标量函数，但BlockRelu会整体处理该块。由于numpy数组和张量的不同，此处无法使用numpy函数。我想知道是否有人可以提供帮助。谢谢。这是我的代码：

import tensorflow as tf
import numpy as np
from tensorflow.python.framework import ops

def block_relu(x):
    for i in range(x.shape[0]):
        if x[i] > 0:
            return x
    return x * 0


def grad_block_relu(x):
    for i in range(x.shape[0]):
        if x[i] > 0:
             return np.ones(x.shape[0])
    return x * 0


# transferring a common function into a numpy function, not needed here
'''
block_relu_np = np.vectorize(block_relu)
grad_block_relu_np = np.vectorize(grad_block_relu)
'''
# numpy uses float64 but tensorflow uses float32
block_relu_np32 = lambda x: block_relu(x).astype(np.float32)
grad_block_relu_np32 = lambda x: grad_block_relu(x).astype(np.float32)


def grad_block_relu_tf(x, name=None):
    with ops.name_scope(name, "grad_block_relu_tf", [x]) as name:
        y = tf.py_func(grad_block_relu_np32, [x], [tf.float32], False, name)
    return y[0]


def my_py_func(func, inp, Tout, stateful=False, name=None, my_grad_func=None):
    # a unique name is required to avoid duplicates:
    random_name = "PyFuncGrad" + str(np.random.randint(0, 1E+8))
    tf.RegisterGradient(random_name)(my_grad_func)
    g = tf.get_default_graph()
    with g.gradient_override_map({"PyFunc": random_name, "PyFuncStateless": random_name}):
        return tf.py_func(func, inp, Tout, stateful=stateful, name=name)


# The gradient function we need to pass to the above my_py_func function takes a special form:
# It needs to take in (an operation, the previous gradients before the operation)
# and propagate(i.e., return) the gradients backward after the operation.
def _block_relu_grad(op, pre_grad):
    x = op.inputs[0]
    cur_grad = grad_block_relu(x)
    next_grad = pre_grad * cur_grad
    return next_grad


def block_relu_tf(x, name=None):
    with ops.name_scope(name, "block_relu_tf", [x]) as name:
        y = my_py_func(block_relu_np32, [x], [tf.float32], stateful=False, name=name, my_grad_func=_block_relu_grad)
    return y[0]


with tf.Session() as sess:
    x = tf.constant([-0.3, 0.005, 0.08, 0.12])
    y = block_relu_tf(x)
    tf.global_variables_initializer().run()
    print(x.eval())
    print(y.eval())
    print(tf.gradients(y, [x])[0].eval())

它将给出一个错误： TypeError：不允许将tf.Tensor用作Python bool。使用if t is not None:代替if t:来测试是否定义了张量，并使用TensorFlow ops（例如tf.cond）执行以张量的值为条件的子图。

Answer 1

我很确定您可以使用标准的Tensorflow函数来实现它：

# input: x
y = tf.scalar_mul( tf.sign( tf.reduce_max( tf.nn.relu(x))), x)

Tensorflow中块激活功能的实现

1 个答案: