TensorFlow中cnn-cifar10样例部分代码详解

import tensorflow as tf

from six.moves import urllib

from tensorflow.models.image.cifar10 import cifar10_input

FLAGS = tf.app.flags.FLAGS#还没理解怎么用

#外部引用cifar10_input文件中的参数
image_size=cifar10_input.image_size

num_classes=cifar10_input.num_classes

num_examples_per_epoch_for_train=cifar10_input.num_examples_per_epoch_for_train

num_examples_per_epoch_for_eval=cifar10_input.num_examples_per_epoch_for_eval

#用于描述训练过程的常数

moving_avreage_decay=0.999
num_epochs_per_decay=350.0#衰减呈阶梯函数，控制衰减周期（阶梯宽度）
learing_rate_decay_factor=0.1#学习率衰减因子
initial_learning_rate=0.1#初始学习率

def variable_on_cpu(name,shape,dtype,initializer):
#一个 context manager,用于为新的op指定要使用的硬件
with tf.device('/cpu:0'):
weight=tf.get_variable(name=name,shape=shape,
initializer=initializer,dtype=dtype)

def variable_on_cpu_with_collection(name,shape,dtype,initializer,wd):
with tf.device('/cpu:0'):
weight=tf.get_variable(name=name,shape=shape,
initializer=tf.truncated_normal_initializer
(stddev=stddev,dtype=dtype))#截断正态分布随机数
if wd is not None:
weight_decay=tf.mul(tf.nn.l2_loss(weight),wd,name='weight_loss')
tf.add_to_collection(name='losses',value=weight_decay)
return weight
'''通过使用指数衰减，来维护变量的滑动均值，当训练模型时，维护训练参数的滑动均值是有好处的，
在测试过程中使用滑动参数比最终训练的数值本身会提高模型的实际性能（准确率）。apply（）方法
会添加trained variables和shadow copies，并添加操作来维护变量的滑动平均值到shadow copies
average  #方法可以访问shadow variables，在创建evaluation model时非常有用。
滑动均值是通过指数衰减计算得到的。shadow variable的初始化值和trained variables相同，其更新公式为
shadow_variable = decay * shadow_variable + (1 - decay) * variable'''
def losses_summary(total_loss):
average_op=tf.train.ExponentialMovingAverage(decay=0.9)#创建一个新的指数滑动均值对象
losses=tf.get_collection(key='losses')#从字典集合中返回关键字'losses'对应的所有变量，包括交叉熵损失和正则项损失
#创建'shadow variables'，并添加维护滑动均值的操作
maintain_averages_op=average_op.apply(losses+[losses+[total_loss]])
##维护变量的滑动均值，返回一个能够更新shadow variables的操作
for i in losses+[total_loss]:
tf.scalar_summary(i.op.name+'_raw', i) #保存变量到Summary缓存对象，以便写入到文件中
tf.scalar_summary(i.op.name, average_op.average(i)) #average() returns the shadow variable for a given variable.
return maintain_averages_op  #返回损失变量的更新操作

def inference(images):
with tf.variable_scope(name_or_scope='conv1')as scope:
weight=variable_on_cpu_with_collection(name='weight',
shape=(5,5,3,64),dtype=tf.float32,stddev=0.05,wd=0.0)
bias=variable_on_cpu_with_collection(name='bias',dtype=tf.float32,initializer=tf.constant_initializer(value=0.0))
#tf.constant_initializer()什么意思
conv1_in=tf.nn.conv2(input=images,filter=weight,strides=(1,1,1,1),padding='SAME')
conv1_in=tf.nn.bias_add(value=conv1_in,bias=bias)
conv1_out=tf.nn.relu(conv1_in)#激励层
pool1=tf.nn.max_pool(value=conv1_out,ksize=(1,3,3,1),strides=(1,2,2,1),padding='SAME')
norm1=tf.nn.lrn(input=pool1,depth_radius=4,bias=1.0,alpha=0.001/9.0,beta=0.75)
with tf.variable_scope(name_or_scope='conv2')as scope:
weight=variable_on_cpu_with_collection(name='weight',
shape=(5,5,64,64),dtype=tf.float32,stddev=0.05,wd=0.0)
bias=variable_on_cpu_with_collection(name='bias',dtype=tf.float32,initializer=tf.constant_initializer(value=0.0))
conv2_in=tf.nn.conv2(input=images,filter=weight,strides=(1,1,1,1),padding='SAME')
conv2_in=tf.nn.bias_add(value=conv2_in,bias=bias)
conv2_out=tf.nn.relu(conv2_in)
norm2=tf.nn.lrn(input=pool2,depth_radius=4,bias=1.0,alpha=0.001/9.0,beta=0.75)
pool2=tf.nn.max_pool(value=conv2_out,ksize=(1,3,3,1),strides=(1,2,2,1),padding='SAME')
reshaped_pool2=tf.reshape(tensor=pool2,shape=(-1,6*6*64))
#shape中的参数为-1是什么意思：：-1，就是缺省值，就是先以你们合适，
#到时总数除以你们几个的乘积，我该是几就是几。

with tf.variable_scope(name_or_scope='fully_connected_layer1') as scope:
weight = variable_on_cpu_with_collection(name='weight',
shape=(6*6*64, 384), dtype=tf.float32,stddev=0.04,wd = 0.004)
bias = variable_on_cpu(name='bias', shape=(384), dtype=tf.float32, initializer=tf.constant_initializer(value=0.1))
fc1_in = tf.matmul(reshaped_pool2, weight)+bias
fc1_out = tf.nn.relu(fc1_in)

with tf.variable_scope(name_or_scope='fully_connected_layer2') as scope:
weight = variable_on_cpu_with_collection(name='weight',
shape=(384, 192),dtype=tf.float32,stddev=0.04,wd=0.004)
bias = variable_on_cpu(name='bias', shape=(192), dtype=tf.float32, initializer=tf.constant_initializer(value=0.1))
fc2_in = tf.matmul(fc1_out, weight)+bias
fc2_out = tf.nn.relu(fc2_in)

with tf.variable_scope(name_or_scope='softmax_layer') as scope:
weight = variable_on_cpu_with_collection(name='weight',
shape=(192, 10),dtype=tf.float32,stddev=1/192,wd=0.0)
bias = variable_on_cpu(name='bias', shape=(10), dtype=tf.float32, initializer=tf.constant_initializer(value=0.0))
classifier_in = tf.matmul(fc2_out, weight)+bias
classifier_out = tf.nn.softmax(classifier_in)

return classifier_in

def loss(logits,label):
labels=tf.cast(x=labels,dtype=tf.int32)#强制类型转换
cross_entropy_loss=tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,labels=labels,name='likelihood_loss')
cross_entropy_loss=tf.reduce_mean(cross_entropy_loss,name='cross_entropy_loss')
#对batch_size长度的向量取平均
tf.add_to_collection(name='losses',value=cross_entropy_loss)
#吧张量cross_entropy_loss添加到字典集合中key='losses'的子集中
return tf.add_n(input=tf.get_collection(key='losses'),name='total_loss')
#tf.add_n（）把所有输入的tensor按照元素相加
#返回字典集合中key='losses'的子集中元素之和

def train(total_loss,global_step):
num_batches_per_epoch=num_examples_per_epoch_for_train/FLAGS.batch_size#求训练块的个数
decay_steps=int(num_batches_per_epoch*num_epochs_per_decay)
#每经过decay_step步训练，衰减lr
lr = tf.train.exponential_decay(learning_rate=initial_learning_rate,
global_step=step,decay_steps=decay_steps,decay_rate=learning_rate_decay_factor, staircase=True)
tf.scalar_summary('learning_rate', lr)
losses_movingaverage_op = losses_summary(total_loss)
#tf.control_dependencies是一个context manager,控制节点执行顺序，先执行control_inputs中的操作，再执行context中的操作
with tf.control_dependencies(control_inputs=[losses_movingaverage_op]):
trainer = tf.train.GradientDescentOptimizer(learning_rate=lr)#梯度下降
gradient_pairs = trainer.compute_gradients(loss=total_loss) #返回计算出的（gradient, variable） pairs计算出相关的梯度
'''tf.train.Optimizer.compute_gradients(loss,var_list=None, gate_gradients=1,aggregation_method=None,
colocate_gradients_with_ops=False, grad_loss=None)对var_list中的变量计算loss的梯度该函数为函数minimize()的第一部分，
返回一个以元组(gradient, variable)组成的列表'''

gradient_update = trainer.apply_gradients(grads_and_vars=gradient_pairs, global_step=step) #返回一步梯度更新操作
'''tf.train.Optimizer.apply_gradients(grads_and_vars, global_step=None, name=None)将计算出的梯度应用到变量上，
是函数minimize()的第二部分，返回一个应用指定的梯度的操作Operation，对global_step做自增操作'''

#num_updates参数用于动态调整衰减率，真实的decay_rate =min(decay, (1 + num_updates) / (10 + num_updates)
variables_average_op = tf.train.ExponentialMovingAverage(decay=moving_average_decay, num_updates=step)

# tf.trainable_variables() 方法返回所有`trainable=True`的变量，列表结构
maintain_variable_average_op = variables_average_op.apply(var_list=tf.trainable_variables())#返回模型参数变量的滑动更新操作
with tf.control_dependencies(control_inputs=[gradient_update, maintain_variable_average_op]):
gradient_update_optimizor = tf.no_op() #Does nothing. Only useful as a placeholder for control edges
return gradient_update_optimizor #返回的是更新后的梯度