tensorlayer.activation 源代码

#! /usr/bin/python
# -*- coding: utf-8 -*-
"""A file containing various activation functions."""

import tensorflow as tf

from tensorlayer.decorators import deprecated

__all__ = [
    'leaky_relu',
    'leaky_relu6',
    'leaky_twice_relu6',
    'lrelu',
    'lrelu6',
    'ltrelu6',
    'ramp',
    'swish',
    'sign',
    'htanh',
    'hard_tanh',
    'pixel_wise_softmax',
]


[文档]def ramp(x, v_min=0, v_max=1, name=None): """Ramp activation function. Reference: [tf.clip_by_value]<https://www.tensorflow.org/api_docs/python/tf/clip_by_value> Parameters ---------- x : Tensor input. v_min : float cap input to v_min as a lower bound. v_max : float cap input to v_max as a upper bound. name : str The function name (optional). Returns ------- Tensor A ``Tensor`` in the same type as ``x``. """ return tf.clip_by_value(x, clip_value_min=v_min, clip_value_max=v_max, name=name)
# @deprecated(date="2018-09-30", instructions="This API is deprecated. Please use as `tf.nn.leaky_relu`")
[文档]def leaky_relu(x, alpha=0.2, name="leaky_relu"): """leaky_relu can be used through its shortcut: :func:`tl.act.lrelu`. This function is a modified version of ReLU, introducing a nonzero gradient for negative input. Introduced by the paper: `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__ The function return the following results: - When x < 0: ``f(x) = alpha_low * x``. - When x >= 0: ``f(x) = x``. Parameters ---------- x : Tensor Support input type ``float``, ``double``, ``int32``, ``int64``, ``uint8``, ``int16``, or ``int8``. alpha : float Slope. name : str The function name (optional). Examples -------- >>> import tensorlayer as tl >>> net = tl.layers.Input([10, 200]) >>> net = tl.layers.Dense(n_units=100, act=lambda x : tl.act.lrelu(x, 0.2), name='dense')(net) Returns ------- Tensor A ``Tensor`` in the same type as ``x``. References ---------- - `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__ """ if not (0 < alpha <= 1): raise ValueError("`alpha` value must be in [0, 1]`") with tf.name_scope(name) as name_scope: x = tf.convert_to_tensor(x, name="features") return tf.maximum(x, alpha * x, name=name_scope)
[文档]def leaky_relu6(x, alpha=0.2, name="leaky_relu6"): """:func:`leaky_relu6` can be used through its shortcut: :func:`tl.act.lrelu6`. This activation function is a modified version :func:`leaky_relu` introduced by the following paper: `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__ This activation function also follows the behaviour of the activation function :func:`tf.nn.relu6` introduced by the following paper: `Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__ The function return the following results: - When x < 0: ``f(x) = alpha_low * x``. - When x in [0, 6]: ``f(x) = x``. - When x > 6: ``f(x) = 6``. Parameters ---------- x : Tensor Support input type ``float``, ``double``, ``int32``, ``int64``, ``uint8``, ``int16``, or ``int8``. alpha : float Slope. name : str The function name (optional). Examples -------- >>> import tensorlayer as tl >>> net = tl.layers.Input([10, 200]) >>> net = tl.layers.Dense(n_units=100, act=lambda x : tl.act.leaky_relu6(x, 0.2), name='dense')(net) Returns ------- Tensor A ``Tensor`` in the same type as ``x``. References ---------- - `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__ - `Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__ """ if not isinstance(alpha, tf.Tensor) and not (0 < alpha <= 1): raise ValueError("`alpha` value must be in [0, 1]`") with tf.name_scope(name) as name_scope: x = tf.convert_to_tensor(x, name="features") return tf.minimum(tf.maximum(x, alpha * x), 6, name=name_scope)
[文档]def leaky_twice_relu6(x, alpha_low=0.2, alpha_high=0.2, name="leaky_relu6"): """:func:`leaky_twice_relu6` can be used through its shortcut: :func:`:func:`tl.act.ltrelu6`. This activation function is a modified version :func:`leaky_relu` introduced by the following paper: `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__ This activation function also follows the behaviour of the activation function :func:`tf.nn.relu6` introduced by the following paper: `Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__ This function push further the logic by adding `leaky` behaviour both below zero and above six. The function return the following results: - When x < 0: ``f(x) = alpha_low * x``. - When x in [0, 6]: ``f(x) = x``. - When x > 6: ``f(x) = 6 + (alpha_high * (x-6))``. Parameters ---------- x : Tensor Support input type ``float``, ``double``, ``int32``, ``int64``, ``uint8``, ``int16``, or ``int8``. alpha_low : float Slope for x < 0: ``f(x) = alpha_low * x``. alpha_high : float Slope for x < 6: ``f(x) = 6 (alpha_high * (x-6))``. name : str The function name (optional). Examples -------- >>> import tensorlayer as tl >>> net = tl.layers.Input([10, 200]) >>> net = tl.layers.Dense(n_units=100, act=lambda x : tl.act.leaky_twice_relu6(x, 0.2, 0.2), name='dense')(net) Returns ------- Tensor A ``Tensor`` in the same type as ``x``. References ---------- - `Rectifier Nonlinearities Improve Neural Network Acoustic Models [A. L. Maas et al., 2013] <https://ai.stanford.edu/~amaas/papers/relu_hybrid_icml2013_final.pdf>`__ - `Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__ """ if not isinstance(alpha_high, tf.Tensor) and not (0 < alpha_high <= 1): raise ValueError("`alpha_high` value must be in [0, 1]`") if not isinstance(alpha_low, tf.Tensor) and not (0 < alpha_low <= 1): raise ValueError("`alpha_low` value must be in [0, 1]`") with tf.name_scope(name) as name_scope: x = tf.convert_to_tensor(x, name="features") x_is_above_0 = tf.minimum(x, 6 * (1 - alpha_high) + alpha_high * x) x_is_below_0 = tf.minimum(alpha_low * x, 0) return tf.maximum(x_is_above_0, x_is_below_0, name=name_scope)
[文档]def swish(x, name='swish'): """Swish function. See `Swish: a Self-Gated Activation Function <https://arxiv.org/abs/1710.05941>`__. Parameters ---------- x : Tensor input. name: str function name (optional). Returns ------- Tensor A ``Tensor`` in the same type as ``x``. """ # TODO: in this case, the beta = 1, but the beta can either be a constant or a trainable parameter with tf.name_scope(name): x = tf.nn.sigmoid(x) * x return x
# @tf.RegisterGradient("QuantizeGrad") # def _sign_grad(unused_op, grad): # return tf.clip_by_value(grad, -1, 1)
[文档]@tf.custom_gradient def sign(x): """Sign function. Clip and binarize tensor using the straight through estimator (STE) for the gradient, usually be used for quantizing values in `Binarized Neural Networks`: https://arxiv.org/abs/1602.02830. Parameters ---------- x : Tensor input. Returns ------- Tensor A ``Tensor`` in the same type as ``x``. References ---------- - `Rectifier Nonlinearities Improve Neural Network Acoustic Models, Maas et al. (2013)` http://web.stanford.edu/~awni/papers/relu_hybrid_icml2013_final.pdf - `BinaryNet: Training Deep Neural Networks with Weights and Activations Constrained to +1 or -1, Courbariaux et al. (2016)` https://arxiv.org/abs/1602.02830 """ def grad(dy): return tf.clip_by_value(dy, -1, 1) return tf.sign(x, name='sign'), grad
# if tf.__version__ > "1.7": # @tf.custom_gradient # def sign(x): # https://www.tensorflow.org/versions/master/api_docs/python/tf/custom_gradient?hl=ES#top_of_page # """Differentiable sign function using sigmoid as the derivation function, # see `tf.sign <https://www.tensorflow.org/api_docs/python/tf/sign>`__ and `tf.custom_gradient # <https://www.tensorflow.org/versions/master/api_docs/python/tf/custom_gradient?hl=ES#top_of_page>`__. # # Parameters # ---------- # x : Tensor # input. # # Returns # ------- # Tensor # A ``Tensor`` in the same type as ``x``. # # """ # tao = tf.nn.sigmoid(x) # def grad(): # return tao * (1 - tao) # return tf.sign(x), grad
[文档]def hard_tanh(x, name='htanh'): """Hard tanh activation function. Which is a ramp function with low bound of -1 and upper bound of 1, shortcut is `htanh`. Parameters ---------- x : Tensor input. name : str The function name (optional). Returns ------- Tensor A ``Tensor`` in the same type as ``x``. """ # with tf.variable_scope("hard_tanh"): return tf.clip_by_value(x, -1, 1, name=name)
[文档]@deprecated(date="2018-06-30", instructions="This API will be deprecated soon as tf.nn.softmax can do the same thing") def pixel_wise_softmax(x, name='pixel_wise_softmax'): """Return the softmax outputs of images, every pixels have multiple label, the sum of a pixel is 1. Usually be used for image segmentation. Parameters ---------- x : Tensor input. - For 2d image, 4D tensor (batch_size, height, weight, channel), where channel >= 2. - For 3d image, 5D tensor (batch_size, depth, height, weight, channel), where channel >= 2. name : str function name (optional) Returns ------- Tensor A ``Tensor`` in the same type as ``x``. Examples -------- >>> outputs = pixel_wise_softmax(network.outputs) >>> dice_loss = 1 - dice_coe(outputs, y_, epsilon=1e-5) References ---------- - `tf.reverse <https://www.tensorflow.org/versions/master/api_docs/python/array_ops.html#reverse>`__ """ with tf.name_scope(name): return tf.nn.softmax(x)
# Alias lrelu = leaky_relu lrelu6 = leaky_relu6 ltrelu6 = leaky_twice_relu6 htanh = hard_tanh