tensorflow: 激活函数(Activation_Functions) 探究

激活函数概念

From TensorFlow - Activation_Functions

在神经网络中,我们有很多的 非线性函数 来作为 激活函数

连续 、平滑

tf.sigmoid(x, name = None) == 1 / (1 + exp(-x))

import numpy as np
import tensorflow as tf

sess = tf.Session()
bn = np.random.normal(0, 5, [3, 5])

print bn.shape, type(bn), ':'
print bn
print
output = tf.nn.sigmoid(bn)
print output.shape, type(output), ':'
print sess.run(output)
(3, 5) <type 'numpy.ndarray'> :
[[ 2.42429203 -1.89521415  4.52536321  2.02200042 -0.46109594]
 [-5.37984794  3.82258344  3.05039891  5.35911657  4.04462726]
 [-3.79266918 -7.12570837  1.74167827 -0.85649631 -3.77669239]]

(3, 5) <class 'tensorflow.python.framework.ops.Tensor'> :
[[  9.18661034e-01   1.30651098e-01   9.89285269e-01   8.83087698e-01
    3.86725869e-01]
 [  4.58738158e-03   9.78596887e-01   9.54799746e-01   9.95316973e-01
    9.82785304e-01]
 [  2.20387197e-02   8.03517003e-04   8.50900112e-01   2.98071887e-01
    2.23857102e-02]]

tf.tanh(x, name = None) == ( exp(x) - exp(-x) ) / ( exp(x) + exp(-x) )

import numpy as np
import tensorflow as tf

sess = tf.Session()
bn = np.random.normal(0, 5, [3, 5])

print bn.shape, type(bn), ':'
print bn
print
output = tf.nn.tanh(bn)
print output.shape, type(output), ':'
print sess.run(output)
(3, 5) <type 'numpy.ndarray'> :
[[-1.43756487 -0.82183219  2.83650212 -0.86855883 -2.54894335]
 [ 2.3639829  -5.23813843  6.94823124 -6.59737671  3.62198313]
 [ 9.15073151  2.82883771 -4.40860502 -5.96409016 -2.74915937]]

(3, 5) <class 'tensorflow.python.framework.ops.Tensor'> :
[[-0.89320646 -0.67606587  0.99314851 -0.70064117 -0.98785491]
 [ 0.98246619 -0.99994361  0.99999816 -0.99999628  0.99857208]
 [ 0.99999998  0.99304304 -0.99970372 -0.9999868  -0.99184608]]

tf.nn.softplus(features, name = None) == log ( exp( features ) + 1)

import numpy as np
import tensorflow as tf

sess = tf.Session()
bn = np.random.normal(0, 5, [3, 5])

print bn.shape, type(bn), ':'
print bn
print
output = tf.nn.softplus(bn)
print output.shape, type(output), ':'
print sess.run(output)
(3, 5) <type 'numpy.ndarray'> :
[[ 2.3897838  -9.86605463 -7.58004249 -4.38702367 -1.44367065]
 [ 7.52588384  6.49497224 -4.37733996 -0.68677868 -2.12110005]
 [-6.35464811 -1.70150615  6.51252343 -0.12833586  4.36898049]]

(3, 5) <class 'tensorflow.python.framework.ops.Tensor'> :
[[  2.47747365e+00   5.19057707e-05   5.10409248e-04   1.23609802e-02
    2.11928637e-01]
 [  7.52642265e+00   6.49648212e+00   1.24805143e-02   4.07592450e-01
    1.13239092e-01]
 [  1.73713721e-03   1.67553530e-01   6.51400705e+00   6.31036601e-01
    4.38156511e+00]]

连续、不平滑

tf.nn.relu(features, name = None) == max (features, 0)

import numpy as np
import tensorflow as tf

sess = tf.Session()
bn = np.random.normal(0, 5, [3, 5])

print bn.shape, type(bn), ':'
print bn
print
output = tf.nn.relu(bn)
print output.shape, type(output), ':'
print sess.run(output)
(3, 5) <type 'numpy.ndarray'> :
[[ 4.34288636 -3.14906286 -5.21796011 -2.77006242 -4.92871322]
 [ 9.07049557 -9.64290379 -5.91523423  1.59385546 -2.04672855]
 [-4.10765782  1.51740207 -0.5572445   8.21818142 -4.67065521]]

(3, 5) <class 'tensorflow.python.framework.ops.Tensor'> :
[[ 4.34288636  0.          0.          0.          0.        ]
 [ 9.07049557  0.          0.          1.59385546  0.        ]
 [ 0.          1.51740207  0.          8.21818142  0.        ]]

tf.nn.relu6(features, name = None) == min ( max(features, 0), 6 )

import numpy as np
import tensorflow as tf

sess = tf.Session()
bn = np.random.normal(0, 5, [3, 5])

print bn.shape, type(bn), ':'
print bn
print
output = tf.nn.relu6(bn)
print output.shape, type(output), ':'
print sess.run(output)
(3, 5) <type 'numpy.ndarray'> :
[[ 6.08205437  7.72360999 -1.62220085  5.41621866  5.8087728 ]
 [-5.07454654  3.85471614  1.44742944  2.77378759  3.61971044]
 [ 5.43383943  1.9598894  -2.5352505  -1.38550512  3.64028622]]

(3, 5) <class 'tensorflow.python.framework.ops.Tensor'> :
[[ 6.          6.          0.          5.41621866  5.8087728 ]
 [ 0.          3.85471614  1.44742944  2.77378759  3.61971044]
 [ 5.43383943  1.9598894   0.          0.          3.64028622]]

tf.nn.bias_add(value, bias, name = None) == value + bias (bias是一维的)

import numpy as np
import tensorflow as tf

sess = tf.Session()
bn = np.random.normal(0, 5, [3, 5])

print bn.shape, type(bn), ':'
print bn
print
output = tf.nn.bias_add(value=bn, bias=np.ones_like(bn[0]))
print output.shape, type(output), ':'
print sess.run(output)
(3, 5) <type 'numpy.ndarray'> :
[[-7.24470546  1.40561024  2.27976912 -6.22879516  4.98934916]
 [-9.75160657  6.78796922  0.60843038 -4.94145474 -0.98402315]
 [-7.02590057  1.98236592  0.85727947  0.08917467 -5.54994355]]

(3, 5) <class 'tensorflow.python.framework.ops.Tensor'> :
[[-6.24470546  2.40561024  3.27976912 -5.22879516  5.98934916]
 [-8.75160657  7.78796922  1.60843038 -3.94145474  0.01597685]
 [-6.02590057  2.98236592  1.85727947  1.08917467 -4.54994355]]

随机正则化

tf.nn.dropout(x, keep_prob, noise_shape = None, seed = None, name = None) == keep_prob概率 的神经元输出值将被放大到原来的 1/keep_prob 倍,其余神经元的输出置 0

import numpy as np
import tensorflow as tf

sess = tf.Session()
bn = np.random.normal(0, 5, [3, 5])

print bn.shape, type(bn), ':'
print bn
print
output = tf.nn.dropout(x=bn, keep_prob=0.5)
print output.shape, type(output), ':'
print sess.run(output)
(3, 5) <type 'numpy.ndarray'> :
[[ -6.63260663   5.18248388  -2.64777118  -0.98104194  -4.21568201]
 [  0.94315835   5.73277238  -0.27942206   0.93593509  10.41087634]
 [  0.18322279   5.72198372   5.00533604  -1.80672579  -2.32201658]]

(3, 5) <class 'tensorflow.python.framework.ops.Tensor'> :
[[ -0.           0.          -5.29554235  -1.96208389  -0.        ]
 [  1.88631669   0.          -0.55884411   1.87187017  20.82175267]
 [  0.          11.44396745   0.          -0.          -0.        ]]


本文参与腾讯云自媒体分享计划,欢迎正在阅读的你也加入,一起分享。

发表于

我来说两句

0 条评论
登录 后参与评论

相关文章

来自专栏计算机视觉战队

深度压缩网络 | 较大程度减少了网络参数存储问题

神经网络是计算密集型和内存密集型,很难使它们用有限的硬件资源去部署在嵌入式系统中。为了解决这种限制,本文引入“深度压缩”,一共有三个阶段的流水线:剪枝、量化和霍...

3555
来自专栏宏伦工作室

动手实现notMNIST数据集图片分类

2003
来自专栏YoungGy

MMD_6b_DecisionTree

overview ? construct 构建决策树的时候需要考虑以下问题: 什么时候停止 如果不停止,那么以什么变量的什么特征构建二叉树 如果停止,那么预测的...

2047
来自专栏CreateAMind

DeepMind时间序列生成模型GTMM - NTM改进

使用vae推理;lstm 时间相关建模;external memory加强记忆;和神经图灵机 可微分计算机等进行比较。

1302
来自专栏CreateAMind

DeepMind时间序列生成模型

使用vae推理;lstm 时间相关建模;external memory加强记忆;和神经图灵机 可微分计算机等进行比较。

1302
来自专栏Pytorch实践

Tensorflow实现部分参数梯度更新

在深度学习中,迁移学习经常被使用,在大数据集上预训练的模型迁移到特定的任务,往往需要保持模型参数不变,而微调与任务相关的模型层。本文主要介绍,使用tensorf...

5192
来自专栏chenjx85的技术专栏

超参数搜索——网格搜索和随机搜索

我们在搜索超参数的时候,如果超参数个数较少(三四个或者更少),那么我们可以采用网格搜素,一种穷尽式的搜索方法。

1313
来自专栏机器学习算法工程师

手把手教你搭建目标检测器-附代码

翻译:刘威威 编辑:祝鑫泉 前 言 本文译自:[http://www.hackevolve.com/create-your...

3494
来自专栏瓜大三哥

基于FPGA的均值滤波(一)

均值滤波的数学表达式列出: ? 由上述公式列出求图像均值的步骤: (1)获取当前窗口所有像素。 (2)计算当前窗口所有像素之和。 (3)将(2)结果除以当前窗口...

3406
来自专栏CreateAMind

DeepMind时间序列生成模型GTMM - NTM改进

903

扫码关注云+社区

领取腾讯云代金券