Maxout & Network in Network

深度学习（二十三）Maxout网络学习

论文笔记 《Maxout Networks》 && 《Network In Network》

Network in Network 笔记

深度学习（二十六）Network In Network学习笔记-ICLR 2014

（Paper）Network in Network网络分析

Network
in Network -- 论文笔记

深度学习方法（十）：卷积神经网络结构变化——Maxout Networks，Network
In Network，Global Average Pooling

Network In Network

network_in_network.py

#!/home/sunnymarkliu/software/miniconda2/bin/python
# _*_ coding: utf-8 _*_

"""
Network in Network(NIN) model implementation example using TensorFlow library.
NIN Paper: Network In Network(https://arxiv.org/abs/1312.4400)
@author: MarkLiu
@time  : 17-3-12 上午9:36
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf
import tensorflow.contrib.slim as slim

class NetworkInNetwork(object):
"""
Network in Network(NIN) model
"""

def __init__(self, input_height, input_width, input_channels, num_classes, activation=tf.nn.relu):
self.input_height = input_height
self.input_width = input_width
self.input_channels = input_channels
self.num_classes = num_classes
self.activation = activation

def create_weight_variable(self, shape, name):
initial = tf.truncated_normal(shape, stddev=0.01)
return tf.Variable(initial, name=name)

def create_bias_variable(self, shape, name):
initial = tf.constant(np.random.rand(), shape=shape)
return tf.Variable(initial, name=name)

def mlp_conv(self, x, kernel_size, stride, num_filters, micro_layer_size, name):
"""
multi layer perceptron convolution.
:param num_filters: number of micro_net filter
:param micro_layer_size: [hidden_layer]
:return:
"""
with tf.variable_scope(name, values=[x]):
# first convolution
net = slim.conv2d(inputs=x, num_outputs=num_filters, kernel_size=[kernel_size, kernel_size],
stride=stride, scope='first_conv', padding='SAME')
# cccp layer
with slim.arg_scope([slim.conv2d], kernel_size=[1, 1], stride=1,
padding='VALID', activation_fn=tf.nn.relu):
for hidden_i, hidden_size in enumerate(micro_layer_size):
net = slim.conv2d(net, hidden_size, scope='hidden_' + str(hidden_i))
return net

def golbal_average_pooling(self, x):
"""
golbal average pooling
:param x: [batch, height, width, channels]
"""
shapes = x.get_shape().as_list()
kernel_height = shapes[1]
kernel_width = shapes[2]
return slim.avg_pool2d(x, kernel_size=[kernel_height, kernel_width], stride=1, padding='VALID',
scope='golbal_average_pooling')

def build_nin_model(self):
# input features
self.x = tf.placeholder(tf.float32, shape=[None, self.input_height, self.input_width, self.input_channels],
name='input_layer')
self.y = tf.placeholder(tf.float32, [None, self.num_classes], name='output_layer')

# learning_rate placeholder
self.learning_rate = tf.placeholder(tf.float32, name='learning_rate')
# dropout layer: keep probability, vgg default value:0.5
self.keep_prob = tf.placeholder(tf.float32, name='keep_prob')
print('x:' + str(self.x.get_shape().as_list()))

self.nin_lay_1 = self.mlp_conv(self.x, kernel_size=11, stride=2, num_filters=96,
micro_layer_size=[96, 96], name='nin_lay_1')
# add dropout
dropout = slim.dropout(self.nin_lay_1, keep_prob=self.keep_prob)
self.maxpooling_1 = slim.max_pool2d(dropout, kernel_size=3, stride=2, padding='SAME')
print('maxpooling_1:' + str(self.maxpooling_1.get_shape().as_list()))

self.nin_lay_2 = self.mlp_conv(self.maxpooling_1, kernel_size=5, stride=1, num_filters=256,
micro_layer_size=[256, 256], name='nin_lay_2')
# add dropout
dropout = slim.dropout(self.nin_lay_2, keep_prob=self.keep_prob)
self.maxpooling_2 = slim.max_pool2d(dropout, kernel_size=3, stride=2, padding='SAME')
print('maxpooling_2:' + str(self.maxpooling_2.get_shape().as_list()))

self.nin_lay_3 = self.mlp_conv(self.maxpooling_2, kernel_size=3, stride=1, num_filters=384,
micro_layer_size=[384, 384], name='nin_lay_3')
# NO dropout
self.maxpooling_3 = slim.max_pool2d(self.nin_lay_3, kernel_size=3, stride=2, padding='SAME')
print('maxpooling_3:' + str(self.maxpooling_3.get_shape().as_list()))

self.nin_lay_4 = self.mlp_conv(self.maxpooling_3, kernel_size=3, stride=1, num_filters=1024,
micro_layer_size=[1024, self.num_classes], name='nin_lay_4')
self.maxpooling_4 = slim.max_pool2d(self.nin_lay_4, kernel_size=3, stride=2, padding='SAME')
print('maxpooling_4:' + str(self.maxpooling_4.get_shape().as_list()))
# golbal average pooling
self.digits = self.golbal_average_pooling(self.nin_lay_4)
self.digits = self.digits[:, 0, 0, :]
print('golbal_average_pooling:' + str(self.digits.get_shape().as_list()))
# softmax
self.read_out_logits = tf.nn.softmax(self.digits)
print('read_out_logits:' + str(self.read_out_logits.get_shape().as_list()))

def init_train_test_op(self):
# some loss functions and all -> total loss
self.loss_function = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=self.y,
logits=self.read_out_logits))
# training op
self.training_op = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss_function)
self.predict_op = tf.arg_max(self.read_out_logits, 1)
# predict
predict_matches = tf.equal(tf.arg_max(self.y, dimension=1),
tf.arg_max(self.read_out_logits, 1))
# accuracy metric
self.accuracy = tf.reduce_mean(tf.cast(predict_matches, tf.float32))

def train(self, x, y, learning_rate, keep_prob=0.5):
feed_dict = {
self.x: x,
self.y: y,
self.keep_prob: keep_prob,
self.learning_rate: learning_rate
}
_, train_loss = self.sess.run([self.training_op, self.loss_function], feed_dict=feed_dict)
train_accuracy = self.get_accuracy(x, y)
return train_loss, train_accuracy

def classify(self, features_x):
feed_dict = {self.x: features_x, self.keep_prob: 1.0}
predict_y, prob = self.sess.run([self.predict_op, self.read_out_logits], feed_dict=feed_dict)
return predict_y, prob

def get_accuracy(self, x, y):
feed_dict = {
self.x: x,
self.y: y,
self.keep_prob: 1.0
}
accuracy = self.sess.run(self.accuracy, feed_dict=feed_dict)
return accuracy

def init(self):
self.build_nin_model()
self.init_train_test_op()
self.sess = tf.Session()
init_op = tf.global_variables_initializer()
self.sess.run(init_op)

network_in_network_test.py

#!/home/sunnymarkliu/software/miniconda2/bin/python
# _*_ coding: utf-8 _*_

"""
@author: MarkLiu
@time  : 17-3-12 下午10:03
"""
import h5py
import utils
from network_in_network import NetworkInNetwork
from datautil import DataWapper

print('load train datas...')

num_classes = 10
train_split = 0.85  # training/validation split

data = h5py.File(utils.train_mnist_2_vggnet_size_file, 'r')
images = data['images'][:]
labels = data['labels'][:]
del data
# split data into training and validation sets
train_samples = int(len(images) * train_split)
train_features = images[:train_samples]
validation_features = images[train_samples:]
del images
train_labels = labels[:train_samples]
validation_labels = labels[train_samples:]
del labels
# Parameters
learning_rate = 0.001
training_epochs = 10
batch_size = 100
display_step = 1
total_batch = int(train_samples / batch_size)

nin = NetworkInNetwork(224, 224, 3, 10)
nin.init()

train_datas = DataWapper(train_features, train_labels)
validate_datas = DataWapper(validation_features, validation_labels)

# training
print('Train model ...')
for epoch in range(0, training_epochs):
avg_loss = 0.
for i in range(0, total_batch):
batch_x, batch_y = train_datas.next_batch(batch_size)
train_loss, train_accuracy = nin.train(batch_x, batch_y, learning_rate, keep_prob=0.8)
avg_loss += train_loss / total_batch
print("train loss = %.9f, train accuracy = %.5f" % (train_loss, train_accuracy))

if epoch % display_step == 0:
batch_x, batch_y = validate_datas.next_batch(batch_size * 2)
accuracy = nin.get_accuracy(x=batch_x, y=batch_y)
print("Epoch: %04d, train loss = %.9f, validation accuracy = %.5f" % (epoch + 1, avg_loss, accuracy))
if epoch % 4 == 0:
learning_rate /= 2

print('Train end.')
print('Predict ...')
print('load test datas...')
data = h5py.File(utils.test_mnist_2_vggnet_size_file, 'r')
test_images = data['images'][:]
test_labels = data['labels'][:]
print('load datas done!')
predict_accuracy = nin.get_accuracy(x=test_images, y=test_labels)
print('predict_accuracy = %.5f' % predict_accuracy)