# 深度学习三人行(第7期)----深度学习之避免过拟合(正则化)

• 提前停止训练
• L1和L2范数正则化
• DroupOut
• 最大范数正则化
• 数据增强
• 小结

1[...] # construct the neural network
2base_loss = tf.reduce_mean(xentropy, name="avg_xentropy")
3reg_losses = tf.reduce_sum(tf.abs(weights1)) + tf.reduce_sum(tf.abs(weights2))
4loss = tf.add(base_loss, scale * reg_losses, name="loss")

1with arg_scope(
2     [fully_connected],weights_regularizer=tf.contrib.layers.l1_regularizer(scale=0.01)):
3     hidden1 = fully_connected(X, n_hidden1, scope="hidden1")
4     hidden2 = fully_connected(hidden1, n_hidden2, scope="hidden2")
5     logits = fully_connected(hidden2, n_outputs, activation_fn=None,scope="out")

1reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
2loss = tf.add_n([base_loss] + reg_losses, name="loss")

1from tensorflow.contrib.layers import dropout
2[...]
3is_training = tf.placeholder(tf.bool, shape=(), name='is_training')
4keep_prob = 0.5
5X_drop = dropout(X, keep_prob, is_training=is_training)
6hidden1 = fully_connected(X_drop, n_hidden1, scope="hidden1")
7hidden1_drop = dropout(hidden1, keep_prob, is_training=is_training)
8hidden2 = fully_connected(hidden1_drop, n_hidden2, scope="hidden2")
9hidden2_drop = dropout(hidden2, keep_prob, is_training=is_training)
10logits = fully_connected(hidden2_drop, n_outputs, activation_fn=None,scope="outputs")

TensorFlow并没有提供一个现成的最大范数正则化函数，但是实施起来也并不麻烦。如下：

1threshold = 1.0
2clipped_weights = tf.clip_by_norm(weights, clip_norm=threshold, axes=1)
3clip_weights = tf.assign(weights, clipped_weights)

1with tf.Session() as sess:
2    [...]
3    for epoch in range(n_epochs):
4        [...]
5        for X_batch, y_batch in zip(X_batches, y_batches):
6            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})
7            clip_weights.eval()

1hidden1 = fully_connected(X, n_hidden1, scope="hidden1")
2with tf.variable_scope("hidden1", reuse=True):
3    weights1 = tf.get_variable("weights")

1hidden1 = fully_connected(X, n_hidden1, scope="hidden1")
2hidden2 = fully_connected(hidden1, n_hidden2, scope="hidden2")
3[...]
4with tf.variable_scope("", default_name="", reuse=True): # root scope
5    weights1 = tf.get_variable("hidden1/weights")
6    weights2 = tf.get_variable("hidden2/weights")

1for variable in tf.global_variables():
2    print(variable.name)

1def max_norm_regularizer(threshold, axes=1, name="max_norm",
2collection="max_norm"):
3    def max_norm(weights):
4        clipped = tf.clip_by_norm(weights, clip_norm=threshold, axes=axes)
5        clip_weights = tf.assign(weights, clipped, name=name)
7        return None # there is no regularization loss term
8    return max_norm

1max_norm_reg = max_norm_regularizer(threshold=1.0)
2hidden1 = fully_connected(X, n_hidden1, scope="hidden1",weights_regularizer=max_norm_reg)

1clip_all_weights = tf.get_collection("max_norm")
2with tf.Session() as sess:
3    [...]
4    for epoch in range(n_epochs):
5        [...]
6        for X_batch, y_batch in zip(X_batches, y_batches):
7        sess.run(training_op, feed_dict={X: X_batch, y: y_batch})
8        sess.run(clip_all_weights)

TensorFlow中提供了一些图像操作的方法，比如平移，旋转，缩放，翻转，剪切，调整光照对比度，饱和度，色度等，这就使得对数据增强比较方便。

333 篇文章66 人订阅

0 条评论

## 相关文章

1.2K80

760100

41470

31020

18730

40370

481100

47450

41030

### 应用：数据预处理-缺失值填充

2.直接根据没有缺失的数据线性回归填充，这样填充的好会共线性，填充的不好就没价值，很矛盾

10430